D-Net

<workflow-app name="lod_generation" xmlns="uri:oozie:workflow:0.4">

    <!-- map reduce job that exports hbase data and prepares them for import

        to the lod_generation -->

    <global>

        <job-tracker>${jobTracker}</job-tracker>

        <name-node>${nameNode}</name-node>

        <configuration>

            <property>

                <name>mapred.job.queue.name</name>

                <value>${queueName}</value>

            </property>

            <property>

                <name>oozie.sqoop.log.level</name>

                <value>DEBUG</value>

            </property>

        </configuration>

    </global>

    <start to='datasetProcessing'/>

    <action name="datasetProcessing">

        <map-reduce>

           <!-- <prepare>

                <delete path="${nameNode}${lod_output}"/>

            </prepare>

-->

            <configuration>

                <property>

                <!-- ZOOKEEPER -->

                <property>

                    <name>hbase.zookeeper.quorum</name>

                    <value>

                        namenode1.hadoop.dm.openaire.eu,namenode2.hadoop.dm.openaire.eu,jobtracker1.hadoop.dm.openaire.eu,jobtracker2.hadoop.dm.openaire.eu,hbase-master1.hadoop.dm.openaire.eu

                    </value>

                    <!-- <value>quorum1.t.hadoop.research-infrastructures.eu,quorum2.t.hadoop.research-infrastructures.eu,quorum3.t.hadoop.research-infrastructures.eu,quorum4.t.hadoop.research-infrastructures.eu,jobtracker.t.hadoop.research-infrastructures.eu

                        </value> -->

                </property>

                <property>

                    <name>zookeeper.znode.rootserver</name>

                    <value>root-region-server</value>

                </property>

                <property>

                    <name>hbase.zookeeper.property.clientPort</name>

                    <value>2181</value>

                </property>

                <!-- MR IO FOR MULTIPLE INPUTS-->

                <property>

                    <name>mapreduce.inputformat.class</name>

                    <value>org.apache.hadoop.mapreduce.lib.input.DelegatingInputFormat</value>

                </property>

                <property>

                    <name>mapreduce.map.class</name>

                    <value>org.apache.hadoop.mapreduce.lib.input.DelegatingMapper</value>

                </property>

                <property>

                    <name>mapreduce.input.multipleinputs.dir.formats</name>

                    <value>${nameNode}${sourceInput};org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat,${nameNode}${targetInput};org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat</value>

                </property>

                <property>

                    <name>mapreduce.input.multipleinputs.dir.mappers</name>

                    <value>${nameNode}${sourceInput};eu.dnetlib.data.mapreduce.hbase.lodExport.SourceMapper,${nameNode}${targetInput};eu.dnetlib.data.mapreduce.hbase.lodExport.TargetMapper</value>

                </property>

                <property>

                    <name>mapred.mapoutput.key.class</name>

                    <value>org.apache.hadoop.io.Text</value>

                </property>

                <property>

                    <name>mapred.mapoutput.value.class</name>

                    <value>org.apache.hadoop.io.Text</value>

                </property>

                <property>

                    <name>mapred.output.key.class</name>

                    <value>org.apache.hadoop.io.Text</value>

                </property>

                <property>

                    <name>mapred.output.value.class</name>

                    <value>org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat</value>

                </property>

                <!-- ## This is required for new MapReduce API usage -->

                <property>

                    <name>mapred.mapper.new-api</name>

                    <value>true</value>

                </property>

                <property>

                    <name>mapred.reducer.new-api</name>

                    <value>true</value>

                </property>

                <!-- # Job-specific options -->

                <property>

                    <name>dfs.blocksize</name>

                    <value>32M</value>

                </property>

                <property>

                    <name>mapred.output.compress</name>

                    <value>false</value>

                </property>

                <property>

                    <name>mapred.reduce.tasks.speculative.execution</name>

                    <value>false</value>

                </property>

                <property>

                    <name>mapred.reduce.tasks.speculative.execution</name>

                    <value>false</value>

                </property>

                <property>

                    <name>mapreduce.map.speculative</name>

                    <value>false</value>

                </property>

                <!-- I/O FORMAT -->

                <!-- IMPORTANT: sets default delimeter used by text output writer. Required

                    to fix issue with traling tab added between id and value in multiple outputs -->

                <property>

                    <name>mapred.textoutputformat.separator</name>

                    <value>${lod_delim}</value>

                </property>

                <!-- ## Names of all output ports -->

<!--

                <property>

                    <name>mapreduce.output</name>

                    <value>

                        ${out1}

                    </value>

                </property>

-->

                <!-- ## Classes of mapper and reducer -->

                <!--<property>

                    <name>mapreduce.map.class</name>

                    <value>eu.dnetlib.data.mapreduce.hbase.lodExport.LodMapper</value>

                </property>

               -->

                <property>

                    <name>mapreduce.reduce.class</name>

                    <value>eu.dnetlib.data.mapreduce.hbase.lodExport.DatasetReducer</value>

                </property>

                <property>

                    <name>io.serializations</name>

                    <value>org.apache.hadoop.io.serializer.WritableSerialization</value>

                </property>

                <!-- ## Custom config -->

                <!--delim character used to seperate fields in hdfs dump files <property> -->

                <property>

                    <name>lod.delim</name>

                    <value>${lod_delim}</value>

                </property>

                <!-- This directory does not correspond to a data store. In fact, this

                    directory only contains multiple data stores. It has to be set to the name

                    of the workflow node. -->

                <property>

                    <name>mapred.output.dir</name>

                    <value>${lod_output}${out}</value>

                </property>

                <!-- ## Workflow node parameters -->

                <property>

                    <name>mapred.reduce.tasks</name>

                    <value>${numReducers}</value>

                </property>

            </configuration>

        </map-reduce>

        <ok to="end"/>

        <error to="fail"/>

    </action>

<!--

    <action name='blocking'>

        <java>

            <prepare>

            </prepare>

            <configuration>

                <property>

                    <name>mapred.job.queue.name</name>

                    <value>${queueName}</value>

                </property>

            </configuration>

            <main-class>eu.dnetlib.iis.core.workflows.lodexport.ClearGraph</main-class>

            <arg>${lod_relationsGraph}</arg>

        </java>

        <ok to="end"/>

        <error to="fail"/>

    </action>

-->

    <!--

    <action name="cleanUpHDFS">

        <fs>

            <delete path="${lod_output}test"/>

        </fs>

        <ok to="end"/>

        <error to="fail"/>

    </action>

-->

    <kill name="fail">

        <message>

            Unfortunately, the process failed -- error message:

            [${wf:errorMessage(wf:lastErrorNode())}]

        </message>

    </kill>

    <end name="end"/>

</workflow-app>