Project

General

Profile

1
<RESOURCE_PROFILE>
2
    <HEADER>
3
        <RESOURCE_IDENTIFIER value="bf0ab07b-36bf-4164-ab73-342bfb11e51a_SGFkb29wSm9iQ29uZmlndXJhdGlvbkRTUmVzb3VyY2VzL0hhZG9vcEpvYkNvbmZpZ3VyYXRpb25EU1Jlc291cmNlVHlwZQ=="/>
4
        <RESOURCE_TYPE value="HadoopJobConfigurationDSResourceType"/>
5
        <RESOURCE_KIND value="HadoopJobConfigurationDSResources"/>
6
        <RESOURCE_URI value=""/>
7
        <DATE_OF_CREATION value="2016-05-23T10:24:25+00:00"/>
8
    </HEADER>
9
    <BODY>
10
        <HADOOP_JOB name="LodGeneration" type="oozie">
11
            <DESCRIPTION>Job for importing data from HBASE to the relational Stats Database</DESCRIPTION>
12
            <STATIC_CONFIGURATION>
13

    
14
                <!-- Cluster wide -->
15
                <PROPERTY key="queueName" value="default"/>
16
                <PROPERTY key="user.name" value="dnet"/> <!-- username = sqoop?? -->
17
                <PROPERTY key="workingDir" value="/lib/lod/releases/working_dir"/>
18

    
19
                <!--Reducers -->
20
                <PROPERTY key="numReducers" value="4"/>
21

    
22
                <!-- Path -->
23
                <PROPERTY key="oozie.wf.application.path" value="hdfs://dm-cluster-nn/lib/stats/releases/stats/oozie_app"/><!-- edit this property! -->
24

    
25
                <!-- DB props-->
26
                <PROPERTY key="Stats_db_Url" value="jdbc:postgresql://stats.openaire.eu:5432/stats"/>
27
                <PROPERTY key="Stats_db_User" value="sqoop"/>
28
                <PROPERTY key="Stats_db_Pass" value="sqoop"/>
29
                <PROPERTY key="Stats_db_Driver" value="org.postgresql.Driver"/>
30

    
31
                <!--Sqoop props-->
32
                <PROPERTY key="Stats_sqoop_RecsPerStatement" value="10000"/>
33
                <PROPERTY key="Stats_sqoop_StatementPerTrans" value="1000000"/>
34
                <PROPERTY key="Stats_sqoop_ReducersCount" value="8"/>
35
                <PROPERTY key="Stats_output_Path" value="/tmp/stats/"/>
36

    
37
                <!--Parser props-->
38
                <PROPERTY key="Stats_db_table_map" value="datasourceLanguage=datasource_languages,datasource=datasource,project=project,result=result,organization=organization,datasourceOrganization=datasource_organizations,datasourceTopic=datasource_topics,projectOrganization=project_organizations,resultClaim=result_claims,resultClassification=result_classifications,resultConcept=result_concepts,resultLanguage=result_languages,resultOrganization=result_organizations,resultResult=result_results,resultProject=project_results,resultResult=result_results,resultTopic=result_topics,category=category,claim=claim,concept=concept,category=category,context=context,datasourceLanguage=datasource_languages,resultLanguage=result_languages,resultDatasource=result_datasources, person=person,personResult=person_results,resultDescription=result_descriptions,datasourceStats=datasource_stats,organizationStats=organization_stats,projectStats=project_stats,resultStats=result_stats,resultOid=result_oids,projectOid=project_oids,personOid=person_oids,datasourceOid=datasource_oids,organizationOid=organization_oids,resultPid=result_pids"/>
39
                <PROPERTY key="Stats_null_String_Field" value="null"/>
40
                <PROPERTY key="Stats_null_Numeric_Field" value="null"/>
41
                <PROPERTY key="Stats_enclosing_Character" value="#"/>
42
                <PROPERTY key="Stats_delim_Character" value="!"/>
43
                <PROPERTY key="Stats_getdeletedbyinference" value="false"/>
44

    
45
                <!--outputs -->
46
                <PROPERTY key="out1" value="datasource"/>
47
                <PROPERTY key="out2" value="project"/>
48
                <PROPERTY key="out3" value="organization"/>
49
                <PROPERTY key="out4" value="datasourceOrganization"/>
50
                <PROPERTY key="out5" value="datasourceTopic"/>
51
                <PROPERTY key="out6" value="datasourceLanguage"/>
52
                <PROPERTY key="out7" value="projectOrganization"/>
53
                <PROPERTY key="out8" value="resultClaim"/>
54
                <PROPERTY key="out9" value="resultClassification"/>
55
                <PROPERTY key="out10" value="resultConcept"/>
56
                <PROPERTY key="out11" value="resultLanguage"/>
57
                <PROPERTY key="out12" value="resultOrganization"/>
58
                <PROPERTY key="out13" value="resultResult"/>
59
                <PROPERTY key="out14" value="resultProject"/>
60
                <PROPERTY key="out15" value="resultTopic"/>
61
                <PROPERTY key="out16" value="resultDatasource"/>
62
                <PROPERTY key="out17" value="result"/>
63
                <PROPERTY key="out18" value="context"/>
64
                <PROPERTY key="out19" value="concept"/>
65
                <PROPERTY key="out20" value="category"/>
66
                <PROPERTY key="out21" value="person"/>
67
                <PROPERTY key="out22" value="personResult"/>
68
                <PROPERTY key="out23" value="resultCitation"/>
69
                <PROPERTY key="out24" value="resultPid"/>
70
                <PROPERTY key="out25" value="resultOid"/>
71
                <PROPERTY key="out26" value="projectOid"/>
72
                <PROPERTY key="out27" value="projectKeyword"/>
73
                <PROPERTY key="out28" value="projectSubject"/>
74
                <PROPERTY key="out29" value="organizationOid"/>
75
                <PROPERTY key="out30" value="datasourceOid"/>
76
                <PROPERTY key="out31" value="personOid"/>
77
                <PROPERTY key="out32" value="projectPerson"/>
78
                <PROPERTY key="out33" value="datasourceStats"/>
79
                <PROPERTY key="out34" value="projectStats"/>
80
                <PROPERTY key="out35" value="organizationStats"/>
81
                <PROPERTY key="out36" value="resultStats"/>
82
                <PROPERTY key="out37" value="person"/>
83
                <PROPERTY key="out38" value="resultDescription"/>
84
                <PROPERTY key="out39" value="organizationOrganization"/>
85

    
86
                <!--usage stats additions-->
87
                <PROPERTY key="piwik_username" value="uminho-pilot"/>
88
                <PROPERTY key="piwik_pass" value="eeTechoo4eeh"/>
89
                <PROPERTY key="piwik_httpProtocol" value="https"/>
90
                <PROPERTY key="piwik_url" value="analytics.openaire.eu"/>
91
                <PROPERTY key="piwik_siteId" value="5"/>
92
                <PROPERTY key="piwik_startDate" value="2015-01-01"/>
93
                <PROPERTY key="piwik_finalDate" value="2016-01-03"/>
94
                <PROPERTY key="piwik_logsPath" value="/tmp/piwikLogs/"/>
95
                <PROPERTY key="piwik_filterOffset" value="5"/>
96
                <PROPERTY key="piwiki_schema" value="usagestats"/>
97
            </STATIC_CONFIGURATION>
98
            <JOB_INTERFACE>
99
                <PARAM description="hdfs name node" name="nameNode" required="true"/>
100
                <PARAM description="job tracker name" name="jobTracker" required="true"/>
101
                <PARAM description="Hbase Table with Protobuffs." name="Stats_Hbase_Source_Table" required="true"/>
102
                <PARAM description="Index Entity Links configuration." name="Stats_indexConf" required="true"/>
103
                <PARAM description="IS lookup service endpoint" name="isLookupEndpoint" required="true"/>
104
            </JOB_INTERFACE>
105
        </HADOOP_JOB>
106
        <STATUS>
107
            <LAST_SUBMISSION_DATE value="2016-05-23T10:43:52+00:00"/>
108
            <RUNNING_INSTANCES value="0"/>
109
            <CUMULATIVE_RUN value="142"/>
110
        </STATUS>
111
        <SECURITY_PARAMETERS>SECURITY_PARAMETERS</SECURITY_PARAMETERS>
112
    </BODY>
113
</RESOURCE_PROFILE>
(7-7/12)