D-Net

<?xml version="1.0"?>

<workflow-app xmlns="uri:oozie:workflow:0.3" name="test-core_examples_javamapreduce_person_by_age_splitter">

	<!-- 

		This example writes to multiple datastores: 2 sequence files with persons of odd and even age. 

		Then another MR job reads both datastores and clones all persons.

	-->

	<start to="data_producer" />

	<action name="data_producer">

		<java>

			<job-tracker>${jobTracker}</job-tracker>

			<name-node>${nameNode}</name-node>

			<!-- The data generated by this node is deleted in this section -->

			<prepare>

				<delete path="${nameNode}${workingDir}/data_producer" />

				<mkdir path="${nameNode}${workingDir}/data_producer" />

			</prepare>

			<configuration>

				<property>

					<name>mapred.job.queue.name</name>

					<value>${queueName}</value>

				</property>

			</configuration>

			<!-- This is simple wrapper for the Java code -->

			<main-class>eu.dnetlib.iis.core.java.ProcessWrapper</main-class>

			<!-- The business Java code that gets to be executed -->

			<arg>eu.dnetlib.iis.core.examples.java.SampleDataProducer</arg>

			<!-- All input and output ports have to be bound to paths in HDFS -->

			<arg>-Operson=${workingDir}/data_producer/person</arg>

			<arg>-Odocument=${workingDir}/data_producer/document</arg>

		</java>

		<ok to="mr_splitter" />

		<error to="fail" />

	</action>

	<action name="mr_splitter">

		<map-reduce>

			<job-tracker>${jobTracker}</job-tracker>

			<name-node>${nameNode}</name-node>

			<!-- The data generated by this node is deleted in this section -->

			<prepare>

				<delete path="${nameNode}${workingDir}/mr_splitter" />

				<!-- multiple output, no need to create mr_splitter, will be created by mapred

				<mkdir path="${nameNode}${workingDir}/mr_splitter" />

				-->

			</prepare>

			<configuration>

				<!-- This is required for new api usage -->

				<property>

					<name>mapred.mapper.new-api</name>

					<value>true</value>

				</property>

				<property>

					<name>mapred.reducer.new-api</name>

					<value>true</value>

				</property>

				<!-- Standard stuff for our framework -->

				<property>

					<name>mapred.job.queue.name</name>

					<value>${queueName}</value>

				</property>

				<property>

					<name>mapred.mapoutput.key.class</name>

					<value>org.apache.hadoop.io.Text</value>

				</property>

				<property>

					<name>mapred.mapoutput.value.class</name>

					<value>org.apache.hadoop.io.BytesWritable</value>

				</property>

				<property>

					<name>mapred.output.key.class</name>

					<value>org.apache.hadoop.io.Text</value>

				</property>

				<property>

					<name>mapred.output.value.class</name>

					<value>org.apache.hadoop.io.BytesWritable</value>

				</property>

				<property>

					<name>mapreduce.inputformat.class</name>

					<value>org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat

					</value>

				</property>

				<property>

					<name>mapreduce.outputformat.class</name>

					<value>org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat

					</value>

				</property>

				<!-- Other stuff -->

				<property>

					<name>mapreduce.map.class</name>

					<value>eu.dnetlib.iis.core.examples.javamapreduce.PersonByAgeSplitter</value>

				</property>

				<property>

					<name>mapred.input.dir</name>

					<value>${workingDir}/data_producer/person</value>

				</property>

				<property>

					<name>mapred.output.dir</name>

					<value>${workingDir}/mr_splitter</value>

				</property>

				<!-- multiple output definitions -->

				<property>

				    <name>named.output.person.age.even</name>

				   <value>${outputAgeEven}</value>

				</property>

				<property>

				    <name>named.output.person.age.odd</name>

				   <value>${outputAgeOdd}</value>

				</property>

			</configuration>

		</map-reduce>

		<ok to="mr_cloner_with_multiple_input" />

		<error to="fail" />

	</action>

    <!-- cloner works on split data, reads from both datastores -->

	<action name="mr_cloner_with_multiple_input">

		<map-reduce>

			<job-tracker>${jobTracker}</job-tracker>

			<name-node>${nameNode}</name-node>

			<!-- The data generated by this node is deleted in this section -->

			<prepare>

				<delete path="${nameNode}${workingDir}/mr_cloner_with_multiple_input" />

				<mkdir path="${nameNode}${workingDir}/mr_cloner_with_multiple_input" />

			</prepare>

			<configuration>

				<!-- This is required for new api usage -->

				<property>

					<name>mapred.mapper.new-api</name>

					<value>true</value>

				</property>

				<property>

					<name>mapred.reducer.new-api</name>

					<value>true</value>

				</property>

				<!-- Standard stuff for our framework -->

				<property>

					<name>mapred.job.queue.name</name>

					<value>${queueName}</value>

				</property>

				<property>

					<name>mapred.mapoutput.key.class</name>

					<value>org.apache.hadoop.io.Text</value>

				</property>

				<property>

					<name>mapred.mapoutput.value.class</name>

					<value>org.apache.hadoop.io.BytesWritable</value>

				</property>

				<property>

					<name>mapred.output.key.class</name>

					<value>org.apache.hadoop.io.Text</value>

				</property>

				<property>

					<name>mapred.output.value.class</name>

					<value>org.apache.hadoop.io.BytesWritable</value>

				</property>

				<property>

					<name>mapreduce.inputformat.class</name>

					<value>org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat

					</value>

				</property>

				<property>

					<name>mapreduce.outputformat.class</name>

					<value>org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat

					</value>

				</property>

				<!-- This is an outdated property -->

				<!-- <property> 

					<name>mapred.map.tasks</name>

					<value>1</value> 

				</property> -->

				<!-- Other stuff -->

				<property>

					<name>mapreduce.map.class</name>

					<value>eu.dnetlib.iis.core.examples.javamapreduce.PersonClonerMapper</value>

				</property>

				<property>

					<name>mapred.input.dir</name>

					<!-- simply setting multiple inputs as CSV -->

					<value>${workingDir}/mr_splitter/${outputAgeEven},${workingDir}/mr_splitter/${outputAgeOdd}</value>

				</property>

				<property>

					<name>mapred.output.dir</name>

					<value>${workingDir}/mr_cloner_with_multiple_input/person</value>

				</property>

				<!-- Workflow node parameters -->

				<property>

					<name>copiesCount</name>

					<value>2</value>

				</property>

			</configuration>

		</map-reduce>

		<ok to="cloner" />

		<error to="fail" />

	</action>

	<action name="cloner">

		<java>

			<job-tracker>${jobTracker}</job-tracker>

			<name-node>${nameNode}</name-node>

			<!-- The data generated by this node is deleted in this section -->

			<prepare>

				<delete path="${nameNode}${workingDir}/cloner" />

				<mkdir path="${nameNode}${workingDir}/cloner" />

			</prepare>

			<configuration>

				<property>

					<name>mapred.job.queue.name</name>

					<value>${queueName}</value>

				</property>

			</configuration>

			<!-- This is simple wrapper for the Java code -->

			<main-class>eu.dnetlib.iis.core.java.ProcessWrapper</main-class>

			<!-- The business Java code that gets to be executed -->

			<arg>eu.dnetlib.iis.core.examples.java.PersonCloner</arg>

			<!-- All input and output ports have to be bound to paths in HDFS -->

			<arg>-Iperson=${workingDir}/mr_cloner_with_multiple_input/person</arg>

			<arg>-Operson=${workingDir}/cloner/person</arg>

		</java>

		<ok to="end" />

		<error to="fail" />

	</action>

	<kill name="fail">

		<message>Unfortunately, the process failed -- error message:

			[${wf:errorMessage(wf:lastErrorNode())}]</message>

	</kill>

	<end name="end" />

</workflow-app>