Project

General

Profile

« Previous | Next » 

Revision 54157

Added by Claudio Atzori over 5 years ago

minor changes

View differences:

modules/dnet-openaireplus-profiles/trunk/src/main/resources/eu/dnetlib/test/profiles/HadoopJobConfigurationDSResources/HadoopJobConfigurationDSResourceType/invalidRecordDoiExporterJob.xml
1 
<RESOURCE_PROFILE>
2 
    <HEADER>
3 
        <RESOURCE_IDENTIFIER value="7edd375c-1d1e-4f2b-81c9-1b7d4f65c233_SGFkb29wSm9iQ29uZmlndXJhdGlvbkRTUmVzb3VyY2VzL0hhZG9vcEpvYkNvbmZpZ3VyYXRpb25EU1Jlc291cmNlVHlwZQ=="/>
4 
        <RESOURCE_TYPE value="HadoopJobConfigurationDSResourceType"/>
5 
        <RESOURCE_KIND value="HadoopJobConfigurationDSResources"/>
6 
        <RESOURCE_URI value=""/>
7 
        <DATE_OF_CREATION value="2018-10-22T10:34:48+00:00"/>
8 
    </HEADER>
9 
    <BODY>
10 
        <HADOOP_JOB name="invalidRecordDoiExporterJob" type="mapreduce">
11 
            <DESCRIPTION>map only job that ensures xml records are valid</DESCRIPTION>
12 
            <STATIC_CONFIGURATION><!-- I/O FORMAT -->
13 
                <PROPERTY key="mapreduce.inputformat.class" value="org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat"/>
14 
                <PROPERTY key="mapreduce.outputformat.class" value="org.apache.hadoop.mapreduce.lib.output.TextOutputFormat"/><!-- MAPPER -->
15 
                <PROPERTY key="mapreduce.map.class" value="eu.dnetlib.data.mapreduce.hbase.dataimport.GetInvalidXmlRecordsMapper"/>
16 
                <PROPERTY key="mapred.mapoutput.key.class" value="org.apache.hadoop.io.Text"/>
17 
                <PROPERTY key="mapred.mapoutput.value.class" value="org.apache.hadoop.io.Text"/><!-- JOB GLOBAL -->
18 
                <PROPERTY key="mapred.output.key.class" value="org.apache.hadoop.io.Text"/>
19 
                <PROPERTY key="mapred.output.value.class" value="org.apache.hadoop.io.Text"/><!-- MISC -->
20 
                <PROPERTY key="mapred.map.tasks.speculative.execution" value="false"/>
21 
                <PROPERTY key="mapreduce.map.speculative" value="false"/>
22 
                <PROPERTY key="mapred.reduce.tasks" value="1"/><!-- <PROPERTY key="user.name" value="dnet" /> --><!--  	Uncomment to override the default lib path -->
23 
                <!-- <PROPERTY key="job.lib" value="/lib/dnet/snapshots/dnet-mapreduce-jobs-assembly-1.1.4-BETA-SNAPSHOT-FilterXmlRecordsMapper.jar"/> -->
24 
                <PROPERTY key="mapred.output.compress" value="false"/>
25 
            </STATIC_CONFIGURATION>
26 
            <JOB_INTERFACE>
27 
                <PARAM description="input sequence file" name="mapred.input.dir" required="true"/>
28 
                <PARAM description="target file" name="mapred.output.dir" required="true"/>
29 
            </JOB_INTERFACE>
30 
        </HADOOP_JOB>
31 
        <STATUS>
32 
            <LAST_SUBMISSION_DATE value="2018-10-22T10:35:08+00:00"/>
33 
            <RUNNING_INSTANCES value="0"/>
34 
            <CUMULATIVE_RUN value="20"/>
35 
        </STATUS>
36 
        <SECURITY_PARAMETERS>SECURITY_PARAMETERS</SECURITY_PARAMETERS>
37 
    </BODY>
38 
</RESOURCE_PROFILE>
modules/dnet-openaireplus-profiles/trunk/src/main/resources/eu/dnetlib/test/profiles/HadoopJobConfigurationDSResources/HadoopJobConfigurationDSResourceType/prepareIndexDataJob.xml
32 32 
				<PROPERTY key="mapred.map.tasks.speculative.execution" value="false" />
33 33 
				<PROPERTY key="mapreduce.map.speculative" value="false" />
34 34 
				<PROPERTY key="mapreduce.reduce.speculative" value="false" />
35
35 

  		




  
  36
  
    
				<PROPERTY key="mapred.fairscheduler.pool" value="solr"/>


  		




  36
  37
  
    
				<PROPERTY key="mapred.reduce.tasks" value="500" />


  		




  37
  38
  
    
				<PROPERTY key="dfs.blocksize" value="256M" />


  		




  38
  39
  
    
		        <PROPERTY key="ui.other.datasourcetypes" value="scholarcomminfra, infospace, pubsrepository::mock, entityregistry, entityregistry::projects, entityregistry::repositories, websource"/>


  		












  

    
      modules/dnet-openaireplus-profiles/trunk/src/main/resources/eu/dnetlib/test/profiles/HadoopJobConfigurationDSResources/HadoopJobConfigurationDSResourceType/xmlRecordValidatorJob.xml
    
  





  
  1
  
    
<RESOURCE_PROFILE>


  		




  
  2
  
    
    <HEADER>


  		




  
  3
  
    
        <RESOURCE_IDENTIFIER value="d051952c-6f2f-4374-ac14-7fbb8d8fa430_SGFkb29wSm9iQ29uZmlndXJhdGlvbkRTUmVzb3VyY2VzL0hhZG9vcEpvYkNvbmZpZ3VyYXRpb25EU1Jlc291cmNlVHlwZQ=="/>


  		




  
  4
  
    
        <RESOURCE_TYPE value="HadoopJobConfigurationDSResourceType"/>


  		




  
  5
  
    
        <RESOURCE_KIND value="HadoopJobConfigurationDSResources"/>


  		




  
  6
  
    
        <RESOURCE_URI value=""/>


  		




  
  7
  
    
        <DATE_OF_CREATION value="2018-10-22T10:34:48+00:00"/>


  		




  
  8
  
    
    </HEADER>


  		




  
  9
  
    
    <BODY>


  		




  
  10
  
    
        <HADOOP_JOB name="xmlRecordValidatorJob" type="mapreduce">


  		




  
  11
  
    
            <DESCRIPTION>map only job that ensures xml records are valid</DESCRIPTION>


  		




  
  12
  
    
            <STATIC_CONFIGURATION><!-- I/O FORMAT -->


  		




  
  13
  
    
                <PROPERTY key="mapreduce.inputformat.class" value="org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat"/>


  		




  
  14
  
    
                <PROPERTY key="mapreduce.outputformat.class" value="org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat"/><!-- MAPPER -->


  		




  
  15
  
    
                <PROPERTY key="mapreduce.map.class" value="eu.dnetlib.data.mapreduce.hbase.dataimport.FilterXmlRecordsMapper"/>


  		




  
  16
  
    
                <PROPERTY key="mapred.mapoutput.key.class" value="org.apache.hadoop.io.Text"/>


  		




  
  17
  
    
                <PROPERTY key="mapred.mapoutput.value.class" value="org.apache.hadoop.io.Text"/><!-- JOB GLOBAL -->


  		




  
  18
  
    
                <PROPERTY key="mapred.output.key.class" value="org.apache.hadoop.io.Text"/>


  		




  
  19
  
    
                <PROPERTY key="mapred.output.value.class" value="org.apache.hadoop.io.Text"/><!-- MISC -->


  		




  
  20
  
    
                <PROPERTY key="mapred.map.tasks.speculative.execution" value="false"/>


  		




  
  21
  
    
                <PROPERTY key="mapreduce.map.speculative" value="false"/>


  		




  
  22
  
    
                <PROPERTY key="mapred.reduce.tasks" value="100"/><!-- <PROPERTY key="user.name" value="dnet" /> --><!--  	Uncomment to override the default lib path -->


  		




  
  23
  
    
                <!-- <PROPERTY key="job.lib" value="/lib/dnet/snapshots/dnet-mapreduce-jobs-assembly-1.1.4-BETA-SNAPSHOT-FilterXmlRecordsMapper.jar"/> -->


  		




  
  24
  
    
                <PROPERTY key="mapred.output.compress" value="true"/>


  		




  
  25
  
    
                <PROPERTY key="mapred.output.compression.type" value="BLOCK"/>


  		




  
  26
  
    
            </STATIC_CONFIGURATION>


  		




  
  27
  
    
            <JOB_INTERFACE>


  		




  
  28
  
    
                <PARAM description="input sequence file" name="mapred.input.dir" required="true"/>


  		




  
  29
  
    
                <PARAM description="target file" name="mapred.output.dir" required="true"/>


  		




  
  30
  
    
            </JOB_INTERFACE>


  		




  
  31
  
    
        </HADOOP_JOB>


  		




  
  32
  
    
        <STATUS>


  		




  
  33
  
    
            <LAST_SUBMISSION_DATE value="2018-10-22T10:35:08+00:00"/>


  		




  
  34
  
    
            <RUNNING_INSTANCES value="0"/>


  		




  
  35
  
    
            <CUMULATIVE_RUN value="20"/>


  		




  
  36
  
    
        </STATUS>


  		




  
  37
  
    
        <SECURITY_PARAMETERS>SECURITY_PARAMETERS</SECURITY_PARAMETERS>


  		




  
  38
  
    
    </BODY>


  		




  
  39
  
    
</RESOURCE_PROFILE>


  		












  

    
      modules/dnet-openaireplus-profiles/trunk/src/main/resources/eu/dnetlib/test/profiles/HadoopJobConfigurationDSResources/HadoopJobConfigurationDSResourceType/informationSpaceConfigurableExportJob.xml
    
  





  
  1
  
    
<RESOURCE_PROFILE>


  		




  
  2
  
    
    <HEADER>


  		




  
  3
  
    
        <RESOURCE_IDENTIFIER value="93078a14-40f4-49f0-9206-a16e6d4e342d_SGFkb29wSm9iQ29uZmlndXJhdGlvbkRTUmVzb3VyY2VzL0hhZG9vcEpvYkNvbmZpZ3VyYXRpb25EU1Jlc291cmNlVHlwZQ=="/>


  		




  
  4
  
    
        <RESOURCE_TYPE value="HadoopJobConfigurationDSResourceType"/>


  		




  
  5
  
    
        <RESOURCE_KIND value="HadoopJobConfigurationDSResources"/>


  		




  
  6
  
    
        <RESOURCE_URI value=""/>


  		




  
  7
  
    
        <DATE_OF_CREATION value="2001-12-31T12:00:00"/>


  		




  
  8
  
    
    </HEADER>


  		




  
  9
  
    
    <BODY>


  		




  
  10
  
    
        <HADOOP_JOB name="informationSpaceConfigurableExportJob" type="mapreduce">


  		




  
  11
  
    
 			<DESCRIPTION>map only job that exports the whole information space table as json</DESCRIPTION>


  		




  
  12
  
    
        	<STATIC_CONFIGURATION>


  		




  
  13
  
    
        	


  		




  
  14
  
    
				<!-- I/O FORMAT -->


  		




  
  15
  
    
				<PROPERTY key="mapreduce.inputformat.class" value="org.apache.hadoop.hbase.mapreduce.TableInputFormat" />


  		




  
  16
  
    
				<PROPERTY key="mapreduce.outputformat.class" value="org.apache.hadoop.mapreduce.lib.output.TextOutputFormat" />


  		




  
  17
  
    
        	


  		




  
  18
  
    
        		<!-- MAPPER -->


  		




  
  19
  
    
				<PROPERTY key="mapreduce.map.class" value="eu.dnetlib.data.mapreduce.hbase.dataexport.ConfigurableExportMapper" />


  		




  
  20
  
    
				<PROPERTY key="mapred.mapoutput.key.class" value="org.apache.hadoop.io.Text" />


  		




  
  21
  
    
				<PROPERTY key="mapred.mapoutput.value.class" value="org.apache.hadoop.io.Text" />


  		




  
  22
  
    
			


  		




  
  23
  
    
			


  		




  
  24
  
    
				<PROPERTY key="mapred.output.key.class" value="org.apache.hadoop.io.Text" />				


  		




  
  25
  
    
				<PROPERTY key="mapred.output.value.class" value="org.apache.hadoop.io.Text" />	


  		




  
  26
  
    
			


  		




  
  27
  
    
				<!-- MISC -->


  		




  
  28
  
    

  		




  
  29
  
    
				<PROPERTY key="mapred.reduce.tasks.speculative.execution" value="false" />	


  		




  
  30
  
    
				<PROPERTY key="mapred.map.tasks.speculative.execution" value="false" />	


  		




  
  31
  
    
				<PROPERTY key="mapreduce.map.speculative" value="false" />


  		




  
  32
  
    
				<PROPERTY key="mapreduce.reduce.speculative" value="false" />	


  		




  
  33
  
    

  		




  
  34
  
    
				<PROPERTY key="dfs.blocksize" value="256M" />


  		




  
  35
  
    
				


  		




  
  36
  
    
				<PROPERTY key="mapred.reduce.tasks" value="1" />


  		




  
  37
  
    
				<!-- <PROPERTY key="user.name" value="dnet" /> -->


  		




  
  38
  
    
				


  		




  
  39
  
    
		<!--  	Uncomment to override the default lib path -->			


  		




  
  40
  
    
		<!--	<PROPERTY key="job.lib" value="/user/dnet/dnet-mapreduce-jobs-0.0.2-SNAPSHOT-jar-with-dependencies.jar"/> -->


  		




  
  41
  
    
        	</STATIC_CONFIGURATION>


  		




  
  42
  
    
        	<JOB_INTERFACE>


  		




  
  43
  
    
        		<PARAM name="hbase.mapred.inputtable" required="true" description="source hbase table" />


  		




  
  44
  
    
        		<PARAM name="hbase.mapreduce.inputtable" required="true" description="source hbase table" />


  		




  
  45
  
    
        		


  		




  
  46
  
    
        		<PARAM name="mapred.output.dir" required="true" description="target sequence file on hdfs" />         		


  		




  
  47
  
    
        	</JOB_INTERFACE>


  		




  
  48
  
    
        	<SCAN>


  		




  
  49
  
    
        		<FILTERS />


  		




  
  50
  
    
        		<FAMILIES />


  		




  
  51
  
    
        	</SCAN>


  		




  
  52
  
    
        </HADOOP_JOB>


  		




  
  53
  
    
        <STATUS>


  		




  
  54
  
    
            <LAST_SUBMISSION_DATE value="2001-12-31T12:00:00"/>


  		




  
  55
  
    
            <RUNNING_INSTANCES value="0"/>


  		




  
  56
  
    
            <CUMULATIVE_RUN value="0" />


  		




  
  57
  
    
        </STATUS>


  		




  
  58
  
    
        <SECURITY_PARAMETERS>SECURITY_PARAMETERS</SECURITY_PARAMETERS>


  		




  
  59
  
    
    </BODY>


  		




  
  60
  
    
</RESOURCE_PROFILE>


  		












Also available in:  Unified diff