D-Net

#!/bin/bash

if [ $# == "0" ] ; then

    oozie job -oozie http://oozie.hadoop.dm.openaire.eu:11000/oozie -config job.properties -run

else

    oozie job -oozie http://oozie.hadoop.dm.openaire.eu:11000/oozie -config $1/job.properties -run

fi

#Sun Jun 29 15:25:26 EEST 2014

storageID=DS-49203724-127.0.0.1-33697-1404044726267

clusterID=testClusterID

cTime=0

storageType=DATA_NODE

layoutVersion=-40

1

#!/bin/bash

echo ""

echo "---->Contents of the working directory"

hadoop fs -ls /user/eri.katsari/core/javamapreduce/lodinter/working_dir

#!/bin/bash

echo ""

echo "---->Contents of the working directory"

hadoop fs -ls /user/eri.katsari/core/javamapreduce/lodinter/working_dir

#!/bin/bash

if [ $# == "0" ] ; then

    oozie job -oozie http://oozie.hadoop.dm.openaire.eu:11000/oozie -config job.properties -run

else

    oozie job -oozie http://oozie.hadoop.dm.openaire.eu:11000/oozie -config $1/job.properties -run

fi

#!/bin/bash

hadoop fs -get /user/eri.katsari/core/javamapreduce/lodinter/working_dir

#Sun Jun 29 15:25:23 EEST 2014

namespaceID=1227977966

clusterID=testClusterID

cTime=0

storageType=NAME_NODE

blockpoolID=BP-283802306-127.0.0.1-1404044723812

layoutVersion=-40

090b992f82e45f2f9ac2ac5328528a4c *fsimage_0000000000000000000

1

# Mon Jan 25 20:56:15 EET 2016

#/log_enable=2

isLookupEndpoint=http://services.openaire.eu:8280/is/services/isLookUp

jobTracker=nmis-hadoop-jt

lod_EntitiesInputFile=/tmp/lod_dump/entities-r-00066

lod_RelationsInputFile=/tmp/lod_dump/relations-r-00042

lod_baseURI=http://lod.openaire.eu/data/

lod_conLine=jdbc:virtuoso://virtuoso-openaire.d4science.org:1111/autoReconnect=true/charset=UTF-8/log_enable=1

lod_dataPath=/user/giorgos.alexiou/rdfData

lod_defaultGraph=test

lod_delim=,

lod_enclosing='

lod_entitiesPerQuery=10

lod_hbase_table=db_openaireplus_services

lod_indexConf=index.conf{ result { dups = true, links = [ { relType = personResult_authorship_hasAuthor, targetEntity = person, expandAs = rel, symmetric = true, fields = [title,dateofacceptance,publisher,resulttype] }, { relType = resultResult_dedup_isMergedIn, targetEntity = result, expandAs = child, symmetric = true, fields = [title,dateofacceptance,publisher,resulttype] }, { relType = resultResult_dedup_merges, targetEntity = result, expandAs = child, symmetric = true, fields = [title,dateofacceptance,publisher,resulttype] }, { relType = resultResult_publicationDataset_isRelatedTo, targetEntity = result, expandAs = rel, symmetric = true, fields = [title,dateofacceptance,publisher,resulttype] }, { relType = resultResult_similarity_isAmongTopNSimilarDocuments, targetEntity = result, expandAs = rel, symmetric = true, fields = [title,dateofacceptance,publisher,resulttype,similarity,type] }, { relType = resultResult_similarity_hasAmongTopNSimilarDocuments, targetEntity = result, expandAs = rel, symmetric = true, fields = [title,dateofacceptance,publisher,resulttype,similarity,type] } ]}, person { dups = false, links = [ { relType = personResult_authorship_isAuthorOf, targetEntity = result, expandAs = rel, symmetric = true, fields = [fullname,ranking] }, { relType = projectPerson_contactPerson_isContact, targetEntity = project, expandAs = rel, symmetric = true, fields = [fullname,email,fax,phone] } ]}, datasource { dups = false, links = [ { relType = datasourceOrganization_provision_provides, targetEntity = organization, expandAs = rel, symmetric = true, fields = [officialname,websiteurl,datasourcetype,aggregatortype] } ]}, organization { dups = false, links = [ { relType = projectOrganization_participation_isParticipant, targetEntity = project, expandAs = rel, symmetric = true, fields = [legalname,legalshortname,websiteurl,country] }, { relType = datasourceOrganization_provision_isProvidedBy, targetEntity = datasource, expandAs = rel, symmetric = true, fields = [legalname,legalshortname,websiteurl,country] }, { relType = organizationOrganization_dedup_merges, targetEntity = organization, expandAs = child, symmetric = true, fields = [legalname,legalshortname,websiteurl,country] }, { relType = organizationOrganization_dedup_isMergedIn, targetEntity = organization, expandAs = child, symmetric = true, fields = [legalname,legalshortname,websiteurl,country] } ]}, project { dups = false, links = [ { relType = projectOrganization_participation_hasParticipant, targetEntity = organization, expandAs = rel, symmetric = true, fields = [code,acronym,title,websiteurl,contracttype,fundingtree] }, { relType = resultProject_outcome_produces, targetEntity = result, expandAs = rel, symmetric = true, fields = [code,acronym,title,websiteurl,contracttype,fundingtree] }, { relType = projectPerson_contactPerson_hasContact, targetEntity = person, expandAs = rel, symmetric = true, fields = [code,acronym,title,websiteurl,contracttype,fundingtree] } ]}}

lod_jsonEntities={ "result": [{ "0": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "1": "http://purl.org/dc/terms/identifier", "2": "http://lod.openaire.eu/vocab/dateOfTransformation", "3": "http://lod.openaire.eu/vocab/dateOfCollection", "4": "http://purl.org/dc/terms/identifier", "5": "http://www.eurocris.org/ontologies/cerif/1.3#name", "6": "http://purl.org/dc/terms/dateAccepted", "7": "http://purl.org/dc/terms/publisher", "8": "http://purl.org/dc/terms/identifier", "9": "http://purl.org/dc/terms/language", "10": "http://purl.org/dc/terms/date", "11": "http://lod.openaire.eu/vocab/resultSubject", "12": "http://lod.openaire.eu/vocab/externalReference", "13": "http://purl.org/dc/terms/source", "14": "http://purl.org/dc/terms/format", "15": "http://lod.openaire.eu/vocab/context", "16": "http://dbpedia.org/ontology/country", "17": "http://purl.org/dc/terms/accessRights", "18": "http://purl.org/dc/terms/description", "19": "http://lsdis.cs.uga.edu/projects/semdis/opus#journal_name", "20": "http://lod.openaire.eu/vocab/dataSourceType", "21": "http://lod.openaire.eu/vocab/device", "22": "http://lod.openaire.eu/vocab/size", "23": "http://lod.openaire.eu/vocab/version", "24": "http://lod.openaire.eu/vocab/lastMetadataUpdate", "25": "http://lod.openaire.eu/vocab/metadataVersion", "26": "http://lod.openaire.eu/vocab/resultType", "27": "http://lod.openaire.eu/vocab/year", "28": "http://lod.openaire.eu/vocab/trust", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": "http://www.eurocris.org/ontologies/cerif/1.3#ResultEntity" }], "person": [{ "0": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "1": "http://purl.org/dc/terms/identifier", "2": "http://lod.openaire.eu/vocab/dateOfTransformation", "3": "http://lod.openaire.eu/vocab/dateOfCollection", "4": "http://purl.org/dc/terms/identifier","5": "http://xmlns.com/foaf/0.1/firstName", "6": "http://xmlns.com/foaf/spec/lastName", "7": "http://xmlns.com/foaf/0.1/name", "8": "http://schema.org/faxNumber", "9": "http://xmlns.com/foaf/0.1/mbox", "10": "http://xmlns.com/foaf/0.1/phone", "11": "http://schema.org/nationality", "12": "http://purl.org/dc/terms/identifier", "13": "http://lod.openaire.eu/vocab/trust", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": "http://xmlns.com/foaf/0.1/Person" }], "datasource": [{ "0": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "1": "http://purl.org/dc/terms/identifier", "2": "http://lod.openaire.eu/vocab/dateOfTransformation", "3": "http://lod.openaire.eu/vocab/dateOfCollection", "4": "http://purl.org/dc/terms/identifier", "5": "http://lod.openaire.eu/vocab/datasourceType", "6": "http://lod.openaire.eu/vocab/openAIRECompatibility", "7": "http://dbpedia.org/ontology/officialName", "8": "http://lod.openaire.eu/vocab/englishName", "9": "http://schema.org/url", "10": "http://xmlns.com/foaf/0.1/logo", "11": "http://xmlns.com/foaf/0.1/mbox", "12": "http://purl.org/vocab/vann/preferredNamespacePrefix", "13": "http://www.w3.org/2003/01/geo/wgs84_pos#lat", "14": "http://www.w3.org/2003/01/geo/wgs84_pos#long", "15": "http://lod.openaire.eu/vocab/dateOfValidity", "16": "http://purl.org/dc/terms/description", "17": "http://lod.openaire.eu/vocab/subjectList", "18": "http://lod.openaire.eu/numberOfItems", "19": "http://purl.org/dc/terms/date", "20": "http://lod.openaire.eu/vocab/policies", "21": "http://lod.openaire.eu/vocab/languages", "22": "http://lod.openaire.eu/vocab/contentType", "23": "http://lod.openaire.eu/vocab/accessInfoPackage", "24": "http://lod.openaire.eu/vocab/releaseStartDate", "25": "http://lod.openaire.eu/vocab/releaseEndDate", "26": "http://lod.openaire.eu/vocab/missionStatementUrl", "27": "http://www.europeana.eu/schemas/edm/dataProvider", "28": "http://lod.openaire.eu/vocab/serviceProvider", "29": "http://lod.openaire.eu/vocab/databaseAccessType", "30": "http://lod.openaire.eu/vocab/dataUploadType", "31": "http://lod.openaire.eu/vocab/dataUploadRestrictions", "32": "http://lod.openaire.eu/vocab/versioning", "33": "http://lod.openaire.eu/vocab/citationGuidelineUrl", "34": "http://lod.openaire.eu/vocab/qualityManagementKind", "35": "http://lod.openaire.eu/vocab/pidSystems", "36": "http://lod.openaire.eu/vocab/certificates", "37": "http://purl.org/dc/terms/accessRights", "38": "http://lod.openaire.eu/vocab/trust", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": "http://www.w3.org/ns/prov#Entity" }], "organization": [{ "0": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "1": "http://purl.org/dc/terms/identifier", "2": "http://lod.openaire.eu/vocab/dateOfTransformation", "3": "http://lod.openaire.eu/vocab/dateOfCollection", "4": "http://purl.org/dc/terms/identifier", "5": "http://www.w3.org/2004/02/skos/core#altLabel", "6": "http://www.w3.org/2004/02/skos/core#prefLabel", "7": "http://lod.openaire.eu/vocab/webSiteUrl", "8": "http://xmlns.com/foaf/0.1/logo", "9": "http://dbpedia.org/ontology/country", "10": "http://lod.openaire.eu/vocab/entityType", "11": "http://lod.openaire.eu/vocab/trust", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": "http://xmlns.com/foaf/0.1/Organization" }], "project": [{ "0": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "1": "http://purl.org/dc/terms/identifier", "2": "http://lod.openaire.eu/vocab/dateOfTransformation", "3": "http://lod.openaire.eu/vocab/dateOfCollection", "4": "http://purl.org/dc/terms/identifier", "5": "http://lod.openaire.eu/vocab/projectCode", "6": "http://schema.org/url", "7": "http://www.eurocris.org/ontologies/cerif/1.3#acronym", "8": "http://www.eurocris.org/ontologies/cerif/1.3#name", "9": "http://www.eurocris.org/ontologies/cerif/1.3#startDate", "10": "http://www.eurocris.org/ontologies/cerif/1.3#endDate", "11": "http://purl.org/cerif/frapo/hasCallIdentifier", "12": "http://www.eurocris.org/ontologies/cerif/1.3#keyword", "13": "http://www.w3.org/2006/time#hasDurationDescription", "14": "http://lod.openaire.eu/vocab/ec_SC39", "15": "http://lod.openaire.eu/vocab/contractType", "16": "http://lod.openaire.eu/vocab/oaMandatePublications", "17": "http://lod.openaire.eu/vocab/projectSubjects", "18": "http://od.openaire.eu/vocab/ec_article29-3", "19": "http://lod.openaire.eu/vocab/funder", "20": "http://lod.openaire.eu/vocab/fundingLevel0", "21": "http://lod.openaire.eu/vocab/fundingLevel1", "22": "http://lod.openaire.eu/vocab/fundingLevel2", "23": "http://lod.openaire.eu/vocab/fundingLevel3", "24": "http://lod.openaire.eu/vocab/trust", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": "http://www.eurocris.org/ontologies/cerif/1.3#Project" }] }

lod_jsonRels={ "resultResult": [{ "property": "http://lod.openaire.eu/vocab/resultResult", "sourceType": "1", "sourceId": "2", "targetType": "3", "targetId": "4" }], "resultProject": [{ "property": "http://lod.openaire.eu/vocab/resultProject", "sourceType": "1", "sourceId": "2", "targetType": "3", "targetId": "4" }], "personResult": [{ "property": "http://lod.openaire.eu/vocab/personResult", "sourceType": "1", "sourceId": "2", "targetType": "3", "targetId": "4" }], "personProject": [{ "property": "http://lod.openaire.eu/vocab/personProject", "sourceType": "1", "sourceId": "2", "targetType": "3", "targetId": "4" }], "personPerson": [{ "property": "http://lod.openaire.eu/vocab/personPerson", "sourceType": "1", "sourceId": "2", "targetType": "3", "targetId": "4" }], "datasourceOrganization": [{ "property": "http://lod.openaire.eu/vocab/datasourceOrganization", "sourceType": "1", "sourceId": "2", "targetType": "3", "targetId": "4" }], "projectOrganization": [{ "property": "http://lod.openaire.eu/vocab/projectOrganization", "sourceType": "1", "sourceId": "2", "targetType": "3", "targetId": "4" }], "organizationOrganization": [{ "property": "http://lod.openaire.eu/vocab/organizationOrganization", "sourceType": "1", "sourceId": "2", "targetType": "3", "targetId": "4" }], "projectPerson": [{ "property": "http://www.eurocris.org/ontologies/cerif/1.3/#linksToPerson", "sourceType": "1", "sourceId": "2", "targetType": "3", "targetId": "4" }], "dedup": [{ "property": "http://www.w3.org/2002/07/owl#sameAs", "sourceType": "1", "sourceId": "2", "targetType": "3", "targetId": "4" }] }

lod_lastExecutionDate=2015-05-26

lod_maxCpart=3

lod_minCpart=1

lod_output=/tmp/lod_dump/

lod_part=5

lod_password=virtramvos

lod_relationsGraph=relationsTest

lod_relationsPerQuery=50

lod_seperator=;

lod_username=dba

numReducers=15

nameNode=hdfs://nmis-hadoop-cluster

oozie.wf.application.path=hdfs://nmis-hadoop-cluster/user/eri.katsari/lod/oozie_app

oozieServiceLoc=http://oozie.t.hadoop.research-infrastructures.eu:11000/oozie

out1=entities

out2=relations

queueName=default

user.name=eri.katsari

workingDir=/user/eri.katsari/core/javamapreduce/lodexport/working_dir

<workflow-app name="lod_generation" xmlns="uri:oozie:workflow:0.4">

    <!-- map reduce job that exports hbase data and prepares them for import

        to the lod_generation -->

    <global>

        <job-tracker>${jobTracker}</job-tracker>

        <name-node>${nameNode}</name-node>

        <configuration>

            <property>

                <name>mapred.job.queue.name</name>

                <value>${queueName}</value>

            </property>

            <property>

                <name>oozie.sqoop.log.level</name>

                <value>DEBUG</value>

            </property>

        </configuration>

    </global>

    <start to='datasetProcessing'/>

    <action name="datasetProcessing">

        <map-reduce>

           <!-- <prepare>

                <delete path="${nameNode}${lod_output}"/>

            </prepare>

-->

            <configuration>

                <property>

                <!-- ZOOKEEPER -->

                <property>

                    <name>hbase.zookeeper.quorum</name>

                    <value>

                        namenode1.hadoop.dm.openaire.eu,namenode2.hadoop.dm.openaire.eu,jobtracker1.hadoop.dm.openaire.eu,jobtracker2.hadoop.dm.openaire.eu,hbase-master1.hadoop.dm.openaire.eu

                    </value>

                    <!-- <value>quorum1.t.hadoop.research-infrastructures.eu,quorum2.t.hadoop.research-infrastructures.eu,quorum3.t.hadoop.research-infrastructures.eu,quorum4.t.hadoop.research-infrastructures.eu,jobtracker.t.hadoop.research-infrastructures.eu

                        </value> -->

                </property>

                <property>

                    <name>zookeeper.znode.rootserver</name>

                    <value>root-region-server</value>

                </property>

                <property>

                    <name>hbase.zookeeper.property.clientPort</name>

                    <value>2181</value>

                </property>

                <!-- MR IO FOR MULTIPLE INPUTS-->

                <property>

                    <name>mapreduce.inputformat.class</name>

                    <value>org.apache.hadoop.mapreduce.lib.input.DelegatingInputFormat</value>

                </property>

                <property>

                    <name>mapreduce.map.class</name>

                    <value>org.apache.hadoop.mapreduce.lib.input.DelegatingMapper</value>

                </property>

                <property>

                    <name>mapreduce.input.multipleinputs.dir.formats</name>

                    <value>${nameNode}${sourceInput};org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat,${nameNode}${targetInput};org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat</value>

                </property>

                <property>

                    <name>mapreduce.input.multipleinputs.dir.mappers</name>

                    <value>${nameNode}${sourceInput};eu.dnetlib.data.mapreduce.hbase.lodExport.SourceMapper,${nameNode}${targetInput};eu.dnetlib.data.mapreduce.hbase.lodExport.TargetMapper</value>

                </property>

                <property>

                    <name>mapred.mapoutput.key.class</name>

                    <value>org.apache.hadoop.io.Text</value>

                </property>

                <property>

                    <name>mapred.mapoutput.value.class</name>

                    <value>org.apache.hadoop.io.Text</value>

                </property>

                <property>

                    <name>mapred.output.key.class</name>

                    <value>org.apache.hadoop.io.Text</value>

                </property>

                <property>

                    <name>mapred.output.value.class</name>

                    <value>org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat</value>

                </property>

                <!-- ## This is required for new MapReduce API usage -->

                <property>

                    <name>mapred.mapper.new-api</name>

                    <value>true</value>

                </property>

                <property>

                    <name>mapred.reducer.new-api</name>

                    <value>true</value>

                </property>

                <!-- # Job-specific options -->

                <property>

                    <name>dfs.blocksize</name>

                    <value>32M</value>

                </property>

                <property>

                    <name>mapred.output.compress</name>

                    <value>false</value>

                </property>

                <property>

                    <name>mapred.reduce.tasks.speculative.execution</name>

                    <value>false</value>

                </property>

                <property>

                    <name>mapred.reduce.tasks.speculative.execution</name>

                    <value>false</value>

                </property>

                <property>

                    <name>mapreduce.map.speculative</name>

                    <value>false</value>

                </property>

                <!-- I/O FORMAT -->

                <!-- IMPORTANT: sets default delimeter used by text output writer. Required

                    to fix issue with traling tab added between id and value in multiple outputs -->

                <property>

                    <name>mapred.textoutputformat.separator</name>

                    <value>${lod_delim}</value>

                </property>

                <!-- ## Names of all output ports -->

<!--

                <property>

                    <name>mapreduce.output</name>

                    <value>

                        ${out1}

                    </value>

                </property>

-->

                <!-- ## Classes of mapper and reducer -->

                <!--<property>

                    <name>mapreduce.map.class</name>

                    <value>eu.dnetlib.data.mapreduce.hbase.lodExport.LodMapper</value>

                </property>

               -->

                <property>

                    <name>mapreduce.reduce.class</name>

                    <value>eu.dnetlib.data.mapreduce.hbase.lodExport.DatasetReducer</value>

                </property>

                <property>

                    <name>io.serializations</name>

                    <value>org.apache.hadoop.io.serializer.WritableSerialization</value>

                </property>

                <!-- ## Custom config -->

                <!--delim character used to seperate fields in hdfs dump files <property> -->

                <property>

                    <name>lod.delim</name>

                    <value>${lod_delim}</value>

                </property>

                <!-- This directory does not correspond to a data store. In fact, this

                    directory only contains multiple data stores. It has to be set to the name

                    of the workflow node. -->

                <property>

                    <name>mapred.output.dir</name>

                    <value>${lod_output}${out}</value>

                </property>

                <!-- ## Workflow node parameters -->

                <property>

                    <name>mapred.reduce.tasks</name>

                    <value>${numReducers}</value>

                </property>

            </configuration>

        </map-reduce>

        <ok to="end"/>

        <error to="fail"/>

    </action>

<!--

    <action name='blocking'>

        <java>

            <prepare>

            </prepare>

            <configuration>

                <property>

                    <name>mapred.job.queue.name</name>

                    <value>${queueName}</value>

                </property>

            </configuration>

            <main-class>eu.dnetlib.iis.core.workflows.lodexport.ClearGraph</main-class>

            <arg>${lod_relationsGraph}</arg>

        </java>

        <ok to="end"/>

        <error to="fail"/>

    </action>

-->

    <!--

    <action name="cleanUpHDFS">

        <fs>

            <delete path="${lod_output}test"/>

        </fs>

        <ok to="end"/>

        <error to="fail"/>

    </action>

-->

    <kill name="fail">

        <message>

            Unfortunately, the process failed -- error message:

            [${wf:errorMessage(wf:lastErrorNode())}]

        </message>

    </kill>

    <end name="end"/>

</workflow-app>

# Wed May 25 23:31:08 EEST 2016

jobTracker=dm-cluster-jt

lod_baseURI=http://lod.openaire.eu/data/

lod_conLine=jdbc:virtuoso://virtuoso-beta.openaire.eu:1111/autoReconnect=true/charset=UTF-8/log_enable=1

lod_dataPath=/user/giorgos.alexiou/rdfData

lod_defaultGraph=test

lod_delim=,

lod_enclosing='

lod_maxCpart=3

lod_minCpart=1

lod_output=/tmp/lod/

lod_password=eiloobi2Ail6Aisi

lod_relationsGraph=relationsTest

lod_relationsPerQuery=170

lod_seperator=;

lod_username=dba

nameNode=hdfs://dm-cluster-nn

numReducers=17

oozie.wf.application.path=hdfs://dm-cluster-nn/user/eri.katsari/lodinter/oozie_app

oozieServiceLoc=http://oozie.hadoop.dm.openaire.eu:11000/oozie

out=dataset

queueName=default

sourceInput=/tmp/lod2/source

targetInput=/tmp/lod2/target

user.name=eri.katsari

workingDir=/user/eri.katsari/core/javamapreduce/lodinter/working_dir

#Sun Jun 29 15:25:23 EEST 2014

namespaceID=1227977966

clusterID=testClusterID

cTime=0

storageType=NAME_NODE

blockpoolID=BP-283802306-127.0.0.1-1404044723812

layoutVersion=-40

090b992f82e45f2f9ac2ac5328528a4c *fsimage_0000000000000000000

#Sun Jun 29 15:25:26 EEST 2014

namespaceID=1227977966

cTime=0

blockpoolID=BP-283802306-127.0.0.1-1404044723812

layoutVersion=-40

date="2014-06-29 15:25:31,760"	 time="1404044731760"	 genstamp="1002"	 id="6739057604294048250"

#Sun Jun 29 15:25:26 EEST 2014

storageID=DS-577722861-127.0.0.1-38993-1404044726267

clusterID=testClusterID

cTime=0

storageType=DATA_NODE

layoutVersion=-40

#Sun Jun 29 15:25:26 EEST 2014

namespaceID=1227977966

cTime=0

blockpoolID=BP-283802306-127.0.0.1-1404044723812

layoutVersion=-40

#Sun Jun 29 15:25:26 EEST 2014

storageID=DS-577722861-127.0.0.1-38993-1404044726267

clusterID=testClusterID

cTime=0

storageType=DATA_NODE

layoutVersion=-40

#Sun Jun 29 15:25:26 EEST 2014

namespaceID=1227977966

cTime=0

blockpoolID=BP-283802306-127.0.0.1-1404044723812

layoutVersion=-40

date="2014-06-29 15:25:31,765"	 time="1404044731765"	 genstamp="1002"	 id="6739057604294048250"

#Sun Jun 29 15:25:26 EEST 2014

storageID=DS-49203724-127.0.0.1-33697-1404044726267

clusterID=testClusterID

cTime=0

storageType=DATA_NODE

layoutVersion=-40

#Sun Jun 29 15:25:26 EEST 2014

namespaceID=1227977966

cTime=0

blockpoolID=BP-283802306-127.0.0.1-1404044723812

layoutVersion=-40

Execute the scripts in the following order:

1. `upload_workflow.sh`

2. `run_workflow.sh`

3. `print_working_dir.sh` or `get_working_dir.sh`

#!/bin/bash

set -x ## print every executed command

if [ $# == "0" ] ; then

    target_dir_root=`pwd`'/oozie_app'

else

    target_dir_root=`readlink -f $1`'/oozie_app'

fi

# initial phase, creating symbolic links to jars in all subworkflows

# currently disabled

#libDir=$target_dir_root'/lib'

#dirs=`find $target_dir_root/* -maxdepth 10 -type d`

#for dir in $dirs

#do

#        if [ -f $dir/workflow.xml ]

#        then

#                echo "creating symbolic links to jars in directory: $dir/lib"

#                if [ ! -d "$dir/lib" ]; then

#                        mkdir $dir/lib

#                fi

#                find $libDir -type f -exec ln -s \{\} $dir/lib \;

#        fi

#done

#uploading

hadoop fs -rm -r /user/eri.katsari/eu/dnetlib/iis/core/javamapreduce/lodexport

hadoop fs -mkdir /user/eri.katsari/eu/dnetlib/iis/core/javamapreduce/lodexport

hadoop fs -mkdir /user/eri.katsari/core/javamapreduce/lodinter/working_dir

hadoop fs -put $target_dir_root /user/eri.katsari/eu/dnetlib/iis/core/javamapreduce/lodexport

#!/bin/bash

hadoop fs -get /user/eri.katsari/core/javamapreduce/lodinter/working_dir

#!/bin/bash

set -x ## print every executed command

if [ $# == "0" ] ; then

    target_dir_root=`pwd`'/oozie_app'

else

    target_dir_root=`readlink -f $1`'/oozie_app'

fi

# initial phase, creating symbolic links to jars in all subworkflows

# currently disabled

#libDir=$target_dir_root'/lib'

#dirs=`find $target_dir_root/* -maxdepth 10 -type d`

#for dir in $dirs

#do

#        if [ -f $dir/workflow.xml ]

#        then

#                echo "creating symbolic links to jars in directory: $dir/lib"

#                if [ ! -d "$dir/lib" ]; then

#                        mkdir $dir/lib

#                fi

#                find $libDir -type f -exec ln -s \{\} $dir/lib \;

#        fi

#done

#uploading

hadoop fs -rm -r /user/eri.katsari/eu/dnetlib/iis/core/javamapreduce/lodexport

hadoop fs -mkdir /user/eri.katsari/eu/dnetlib/iis/core/javamapreduce/lodexport

hadoop fs -mkdir /user/eri.katsari/core/javamapreduce/lodinter/working_dir

hadoop fs -put $target_dir_root /user/eri.katsari/eu/dnetlib/iis/core/javamapreduce/lodexport

Execute the scripts in the following order:

1. `upload_workflow.sh`

2. `run_workflow.sh`

3. `print_working_dir.sh` or `get_working_dir.sh`

lod_baseURI=http://lod.openaire.eu/data/

lod_dataPath=/user/giorgos.alexiou/rdfData

lod_delim=,

lod_enclosing='

lod_maxCpart=3

lod_minCpart=1

lod_output=/tmp/lod/

 #---------config for CNR------------

#lod_conLine=jdbc:virtuoso://virtuoso-openaire.d4science.org:1111/autoReconnect=true/charset=UTF-8/log_enable=1

#lod_password=virtramvos

#Config for DM

lod_conLine=jdbc:virtuoso://virtuoso-beta.openaire.eu:1111/autoReconnect=true/charset=UTF-8/log_enable=1

lod_password=eiloobi2Ail6Aisi

lod_defaultGraph=test

lod_relationsGraph=relationsTest

lod_relationsPerQuery=170

lod_seperator=;

lod_username=dba

#--------DM Cluster config-------

jobTracker=dm-cluster-jt

nameNode=hdfs://dm-cluster-nn

oozie.wf.application.path=hdfs://dm-cluster-nn/user/eri.katsari/lodinter/oozie_app

#oozie.wf.application.path=hdfs://dm-cluster-nn/user/giorgos.alexiou/lod/oozie_app

oozieServiceLoc=http://oozie.hadoop.dm.openaire.eu:11000/oozie

#--------CNR cluster config-------

#jobTracker=nmis-hadoop-jt

#nameNode=hdfs://nmis-hadoop-cluster

#oozie.wf.application.path=hdfs://nmis-hadoop-cluster/user/eri.katsari/lod/oozie_app

#oozieServiceLoc=http://oozie.t.hadoop.research-infrastructures.eu:11000/oozie

numReducers=17

out=dataset

sourceInput=/tmp/lod2/source

targetInput=/tmp/lod2/target

queueName=default

#user.name=giorgos.alexiou

#workingDir=/user/giorgos.alexiou/core/javamapreduce/lodexport/working_dir

user.name=eri.katsari

workingDir=/user/eri.katsari/core/javamapreduce/lodinter/working_dir

<?xml version="1.0" encoding="UTF-8"?>

<!--

   Licensed to the Apache Software Foundation (ASF) under one or more

   contributor license agreements.  See the NOTICE file distributed with

   this work for additional information regarding copyright ownership.

   The ASF licenses this file to You under the Apache License, Version 2.0

   (the "License"); you may not use this file except in compliance with

   the License.  You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software

   distributed under the License is distributed on an "AS IS" BASIS,

   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

   See the License for the specific language governing permissions and

   limitations under the License.

-->

<!--

NOTE - This POM is used for Cloudera's build and packaging process.

-->

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

  <modelVersion>4.0.0</modelVersion>

  <parent>

    <groupId>com.cloudera.cdh</groupId>

    <artifactId>hadoop-root</artifactId>

    <version>2.0.0-mr1-cdh4.3.1</version>

    <relativePath>cloudera/maven-packaging/pom.xml</relativePath>

  </parent>

  <groupId>com.cloudera.cdh</groupId>

  <artifactId>hadoop-ant</artifactId>

  <version>2.0.0-mr1-cdh4.3.1</version>

  <packaging>pom</packaging>

  <description>Maven wrapper for Ant Hadoop build</description>

  <name>CDH Hadoop Maven Wrapper</name>

  <profiles>

    <!--To allow standalone project to fetch CDH artifacts -->

    <!--Activate by default doing a negation of negation   -->

    <!-- this trick allows CDH root to switch it off       -->

    <profile>

      <id>cdh.repos</id>

      <activation>

        <activeByDefault>false</activeByDefault>

        <property>

          <name>use.cdh.repos</name>

          <value>!false</value>

        </property>

      </activation>

      <repositories>

        <repository>

          <id>cdh.releases.repo</id>

          <url>https://repository.cloudera.com/content/groups/cdh-releases-rcs</url>

          <name>CDH Releases Repository</name>

          <snapshots>

            <enabled>false</enabled>

          </snapshots>

        </repository>

        <repository>

          <id>cdh.snapshots.repo</id>

          <url>https://repository.cloudera.com/content/repositories/snapshots</url>

          <name>CDH Snapshots Repository</name>

          <snapshots>

            <enabled>true</enabled>

          </snapshots>

        </repository>

      </repositories>

    </profile>

    <!-- Similar logic to allow us to skip compilation/tests when run -->

    <!-- via do-release-build or packaging scripts, since we're already -->

    <!-- compiling via direct ant calls. -->

    <profile>

      <id>non.release.build</id>

      <activation>

        <activeByDefault>false</activeByDefault>

        <property>

          <name>not.cdh.release.build</name>

          <value>!false</value>

        </property>

      </activation>

      <build>

        <plugins>

          <plugin>

            <groupId>org.apache.maven.plugins</groupId>

            <artifactId>maven-antrun-plugin</artifactId>

            <version>1.6</version>

            <inherited>false</inherited>

            <executions>

              <execution>

                <id>clean</id>

                <configuration>

                  <target>

                    <exec executable="${ant.cmd}" dir="${basedir}" failonerror="true">

                      <arg value="clean"/>

                      <arg value="-Dcdh.maven=true"/>

                      <arg value="-Divy.cache.dir=${ivy.cache.dir}"/>

                    </exec>

                  </target>

                </configuration>

                <goals>

                  <goal>run</goal>

                </goals>

                <phase>clean</phase>

              </execution>

              <execution>

                <id>compile</id>

                <configuration>

                  <target>

                    <exec executable="${ant.cmd}" dir="${basedir}" failonerror="true">

                      <arg value="bin-package"/>

                      <arg value="-Dcdh.maven=true"/>

                      <arg value="-Divy.cache.dir=${ivy.cache.dir}"/>

                    </exec>

                  </target>

                </configuration>

                <goals>

                  <goal>run</goal>

                </goals>

                <phase>compile</phase>

              </execution>

              <execution>

                <id>test</id>

                <configuration>

                  <target unless="${skipTests}">

                    <exec executable="${ant.cmd}" dir="${basedir}" failonerror="true">

                      <arg value="test"/>

                      <arg value="-Dcdh.maven=true"/>

                      <arg value="-Divy.cache.dir=${ivy.cache.dir}"/>

                </exec>

                  </target>

                </configuration>

                <goals>

                  <goal>run</goal>

                </goals>

                <phase>test</phase>

              </execution>

            </executions>

          </plugin>

        </plugins>

      </build>

    </profile>

        <!-- Profile that's only activated when we're calling as a release build, to run -->

    <!-- with variables set *and* tar target -->

    <profile>

      <id>cdh.release.build</id>

      <activation>

        <activeByDefault>false</activeByDefault>

        <property>

          <name>not.cdh.release.build</name>

          <value>false</value>

        </property>

      </activation>

      <build>

        <plugins>

          <plugin>

            <groupId>org.apache.maven.plugins</groupId>

            <artifactId>maven-antrun-plugin</artifactId>

            <inherited>false</inherited>

            <executions>

              <execution>

                <id>tar</id>

                <configuration>

                  <target if="do.full.compile">

                    <property file="${basedir}/build.properties"/>

                    <exec executable="/bin/bash" dir="${basedir}/cloudera" failonerror="true">

                      <arg value="do-release-build" />

                      <env key="SKIP_MVN_EXPLICIT" value="true" />

                    </exec>

                  </target>

                </configuration>

                <goals>

                  <goal>run</goal>

                </goals>

                <phase>compile</phase>

              </execution>

            </executions>

          </plugin>

        </plugins>

      </build>

    </profile>

  </profiles>

  <build>

    <plugins>

      <plugin>

        <groupId>com.cloudera.kitchen</groupId>

        <artifactId>maven-deps-to-props-plugin</artifactId>

        <executions>

          <execution>

            <id>build.properties</id>

            <configuration>

              <templateFile>${basedir}/cloudera/maven-packaging/templates/build.template</templateFile>

              <targetFile>${basedir}/build.properties</targetFile>

            </configuration>

            <goals>

              <goal>generate</goal>

            </goals>

            <phase>process-resources</phase>

          </execution>

        </executions>

      </plugin>

    </plugins>

  </build>

</project>