Project

General

Profile

« Previous | Next » 

Revision 57725

Added by Miriam Baglioni over 4 years ago

hadoop profiles for the orcid propagation

View differences:

modules/dnet-openaireplus-profiles/trunk/src/main/resources/eu/dnetlib/test/profiles/HadoopJobConfigurationDSResources/HadoopJobConfigurationDSResourceType/propagationOrcidToResultSaveToFileJob.xml
1 
<RESOURCE_PROFILE>
2 
<HEADER>
3 
    <RESOURCE_IDENTIFIER value="8a2b5e8c-5b92-44fb-b4d5-b0a2e08ac143_SGFkb29wSm9iQ29uZmlndXJhdGlvbkRTUmVzb3VyY2VzL0hhZG9vcEpvYkNvbmZpZ3VyYXRpb25EU1Jlc291cmNlVHlwZQ=="/>
4 
    <RESOURCE_TYPE value="HadoopJobConfigurationDSResourceType"/>
5 
    <RESOURCE_KIND value="HadoopJobConfigurationDSResources"/>
6 
    <RESOURCE_URI value=""/>
7 
    <DATE_OF_CREATION value="2019-10-13T12:34:20+00:00"/>
8 
</HEADER>
9 
<BODY>
10 
    <HADOOP_JOB name="propagationOrcidToResultSaveToFile" type="mapreduce">
11 
        <DESCRIPTION>map reduce job that propagates orcid to results</DESCRIPTION>
12 
        <STATIC_CONFIGURATION><!-- I/O FORMAT -->
13 
            <PROPERTY key="mapreduce.inputformat.class" value="org.apache.hadoop.hbase.mapreduce.TableInputFormat"/>
14 
            <PROPERTY key="mapreduce.outputformat.class" value="org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat"/><!-- MAPPER -->
15 
            <PROPERTY key="mapreduce.map.class" value="eu.dnetlib.data.mapreduce.hbase.propagation.orcidthroughproducts.PropagationOrcidToResultMapper">
16 
            <PROPERTY key="mapred.mapoutput.key.class" value="org.apache.hadoop.hbase.io.ImmutableBytesWritable"/>
17 
            <PROPERTY key="mapred.mapoutput.value.class" value="org.apache.hadoop.io.Text"/><!-- REDUCER -->
18 
            <PROPERTY key="mapreduce.reduce.class" value="eu.dnetlib.data.mapreduce.hbase.propagation.orcidthroughproducts.PropagationOrcidToResultFileReducer"/>
19 
            <PROPERTY key="mapred.output.key.class" value="org.apache.hadoop.io.Text"/>
20 
            <PROPERTY key="mapred.output.value.class" value="org.apache.hadoop.io.Text"/><!-- MISC -->
21 
            <PROPERTY key="mapred.compress.map.output" value="true"/>
22 
            <PROPERTY key="mapred.map.tasks.speculative.execution" value="false"/>
23 
            <PROPERTY key="mapreduce.map.speculative" value="false"/>
24 
            <PROPERTY key="mapreduce.reduce.speculative" value="false"/>
25 
            <PROPERTY key="mapred.reduce.tasks.speculative.execution" value="false"/>
26 
            <PROPERTY key="mapred.reduce.tasks" value="1"/><!-- <PROPERTY key="user.name" value="dnet" /> --><!--  	Uncomment to override the default lib path --><!--<PROPERTY key="job.lib" value="/user/dnet/openaire/dnet-mapreduce-jobs-assembly-country_propagation-1.1.1-SNAPSHOT.jar"/>-->
27 
            <PROPERTY key="job.lib" value="/lib/dnet/snapshots/propagation_orcid2results.jar"/>
28 
        </STATIC_CONFIGURATION>
29 
        <JOB_INTERFACE>
30 
            <PARAM description="source hbase table" name="hbase.mapred.inputtable" required="true"/>
31 
            <PARAM description="target hbase table" name="hbase.mapred.outputtable" required="true"/>
32 
            <PARAM description="source hbase table" name="hbase.mapreduce.inputtable" required="true"/>
33 
            <PARAM description="target hbase table" name="hbase.mapreduce.outputtable" required="true"/><!--<PARAM name="countryPropagation.conf" required="true" description="configuration for country propagation" />-->
34 
            <PARAM description="target sequence file on hdfs" name="mapred.output.dir" required="true"/>
35 
            <!--add needed param to get association between community and organizationtoresult -->
36 
            <!--<PARAM description="map organizationtoresult to communities" name="organizationtoresult.community.map" required="true"/>-->
37 
        </JOB_INTERFACE>
38 
    </HADOOP_JOB>
39 
    <STATUS>
40 
        <LAST_SUBMISSION_DATE value="2001-12-31T12:00:00"/>
41 
        <RUNNING_INSTANCES value="0"/>
42 
        <CUMULATIVE_RUN value="0"/>
43 
    </STATUS>
44 
    <SECURITY_PARAMETERS>SECURITY_PARAMETERS</SECURITY_PARAMETERS>
45 
</BODY>
46 
</RESOURCE_PROFILE>
modules/dnet-openaireplus-profiles/trunk/src/main/resources/eu/dnetlib/test/profiles/HadoopJobConfigurationDSResources/HadoopJobConfigurationDSResourceType/propagationORCIDToResultJob.xml
1 
<RESOURCE_PROFILE>
2 
    <HEADER>
3 
        <RESOURCE_IDENTIFIER value="a971e717-2ebf-44c3-abc7-d5599ad7a86a_SGFkb29wSm9iQ29uZmlndXJhdGlvbkRTUmVzb3VyY2VzL0hhZG9vcEpvYkNvbmZpZ3VyYXRpb25EU1Jlc291cmNlVHlwZQ=="/>
4 
        <RESOURCE_TYPE value="HadoopJobConfigurationDSResourceType"/>
5 
        <RESOURCE_KIND value="HadoopJobConfigurationDSResources"/>
6 
        <RESOURCE_URI value=""/>
7 
        <DATE_OF_CREATION value="2019-08-07T14:39:58+00:00"/>
8 
    </HEADER>
9 
    <BODY>
10 
        <HADOOP_JOB name="propagationOrcidToResult" type="mapreduce">
11 
            <DESCRIPTION>map reduce job that propagates result to a community through the affiliation to organizations</DESCRIPTION>
12 
            <STATIC_CONFIGURATION><!-- I/O FORMAT -->
13 
                <PROPERTY key="mapreduce.inputformat.class" value="org.apache.hadoop.hbase.mapreduce.TableInputFormat"/>
14 
                <PROPERTY key="mapreduce.outputformat.class" value="org.apache.hadoop.hbase.mapreduce.TableOutputFormat"/><!-- MAPPER -->
15 
                <PROPERTY key="mapreduce.map.class" value="eu.dnetlib.data.mapreduce.hbase.propagation.orcidthroughproducts.PropagationOrcidToResultMapper"/>
16 
                <PROPERTY key="mapred.mapoutput.key.class" value="org.apache.hadoop.hbase.io.ImmutableBytesWritable"/>
17 
                <PROPERTY key="mapred.mapoutput.value.class" value="org.apache.hadoop.io.Text"/><!-- REDUCER -->
18 
                <PROPERTY key="mapreduce.reduce.class" value="eu.dnetlib.data.mapreduce.hbase.propagation.orcidthroughproducts.PropagationOrcidToResultReducer"/>
19 
                <PROPERTY key="mapred.output.key.class" value="org.apache.hadoop.hbase.io.ImmutableBytesWritable"/>
20 
                <PROPERTY key="mapred.output.value.class" value="org.apache.hadoop.io.Text"/><!-- MISC -->
21 
                <PROPERTY key="mapred.compress.map.output" value="true"/>
22 
                <PROPERTY key="mapred.map.tasks.speculative.execution" value="false"/>
23 
                <PROPERTY key="mapreduce.map.speculative" value="false"/>
24 
                <PROPERTY key="mapreduce.reduce.speculative" value="false"/>
25 
                <PROPERTY key="mapred.reduce.tasks.speculative.execution" value="false"/>
26 
                <PROPERTY key="mapred.reduce.tasks" value="20"/><!-- <PROPERTY key="user.name" value="dnet" /> --><!--  	Uncomment to override the default lib path --><!--<PROPERTY key="job.lib" value="/user/dnet/openaire/dnet-mapreduce-jobs-assembly-country_propagation-1.1.1-SNAPSHOT.jar"/>-->
27 
                <PROPERTY key="job.lib" value="/lib/dnet/snapshots/propagation_orcid2results.jar"/>
28 
            </STATIC_CONFIGURATION>
29 
            <JOB_INTERFACE>
30 
                <PARAM description="source hbase table" name="hbase.mapred.inputtable" required="true"/>
31 
                <PARAM description="target hbase table" name="hbase.mapred.outputtable" required="true"/>
32 
                <PARAM description="source hbase table" name="hbase.mapreduce.inputtable" required="true"/>
33 
                <PARAM description="target hbase table" name="hbase.mapreduce.outputtable" required="true"/>
34 

  		




  
  35
  
    

  		




  
  36
  
    

  		




  
  37
  
    
            </JOB_INTERFACE>


  		




  
  38
  
    
            <SCAN>


  		




  
  39
  
    
                <FILTERS/>


  		




  
  40
  
    
                <FAMILIES/>


  		




  
  41
  
    
            </SCAN>


  		




  
  42
  
    
        </HADOOP_JOB>


  		




  
  43
  
    
        <STATUS>


  		




  
  44
  
    
            <LAST_SUBMISSION_DATE value="2019-08-07T14:50:14+00:00"/>


  		




  
  45
  
    
            <RUNNING_INSTANCES value="2"/>


  		




  
  46
  
    
            <CUMULATIVE_RUN value="34"/>


  		




  
  47
  
    
        </STATUS>


  		




  
  48
  
    
        <SECURITY_PARAMETERS>SECURITY_PARAMETERS</SECURITY_PARAMETERS>


  		




  
  49
  
    
    </BODY>


  		




  
  50
  
    
</RESOURCE_PROFILE>


  		












Also available in:  Unified diff