Project

General

Profile

« Previous | Next » 

Revision 58291

WIP: data provision workflow exploiting the oozie jobs on the OCEAN Hadoop cluster

View differences:

modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/ocean.provision.meta.xml
1
<RESOURCE_PROFILE>
2
    <HEADER>
3
        <RESOURCE_IDENTIFIER value="d511a255-6356-4116-a862-5c1cc4ebd63c_TWV0YVdvcmtmbG93RFNSZXNvdXJjZXMvTWV0YVdvcmtmbG93RFNSZXNvdXJjZVR5cGU="/>
4
        <RESOURCE_TYPE value="MetaWorkflowDSResourceType"/>
5
        <RESOURCE_KIND value="MetaWorkflowDSResources"/>
6
        <RESOURCE_URI value=""/>
7
        <DATE_OF_CREATION value="2020-02-18T15:01:54+00:00"/>
8
    </HEADER>
9
    <BODY>
10
        <METAWORKFLOW_NAME family="OpenAIRE Provision [OCEAN]">OpenAIRE Provision [OCEAN]</METAWORKFLOW_NAME>
11
        <METAWORKFLOW_DESCRIPTION/>
12
        <METAWORKFLOW_SECTION>InfoSpace Provision</METAWORKFLOW_SECTION>
13
        <ADMIN_EMAIL>alessia.bardi@isti.cnr.it,claudio.atzori@isti.cnr.it</ADMIN_EMAIL>
14
        <CONFIGURATION status="EXECUTABLE">
15
            <WORKFLOW id="74d90d54-bea4-4a79-82d9-adddcc89e660_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl" name="provision">
16
            </WORKFLOW>
17
        </CONFIGURATION>
18
        <SCHEDULING enabled="false">
19
            <CRON>29 5 22 ? * *</CRON>
20
            <MININTERVAL>10080</MININTERVAL>
21
        </SCHEDULING>
22
    </BODY>
23
</RESOURCE_PROFILE>
modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/ocean.provision.xml
1
<RESOURCE_PROFILE>
2
	<HEADER>
3
		<RESOURCE_IDENTIFIER value="74d90d54-bea4-4a79-82d9-adddcc89e660_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl"/>
4
		<RESOURCE_TYPE value="WorkflowDSResourceType"/>
5
		<RESOURCE_KIND value="WorkflowDSResources"/>
6
		<RESOURCE_URI value=""/>
7
		<DATE_OF_CREATION value="2014-08-01T18:13:51.0Z"/>
8
	</HEADER>
9
	<BODY>
10
		<WORKFLOW_NAME>Data Provision [OCEAN]</WORKFLOW_NAME>
11
		<WORKFLOW_TYPE>Data Provision</WORKFLOW_TYPE>
12
		<WORKFLOW_PRIORITY>30</WORKFLOW_PRIORITY>
13
		<CONFIGURATION start="manual">
14

  
15
			<!-- PREPARE NODES -->
16
			<NODE name="setGraphBasePath" type="SetEnvParameter" isStart="true">
17
				<DESCRIPTION>Set the target directory to materialize the graph</DESCRIPTION>
18
				<PARAMETERS>
19
					<PARAM managedBy="system" name="parameterName" required="true" type="string">graphBasePath</PARAM>
20
					<PARAM managedBy="user" name="parameterValue" required="true" type="string">/var/lib/dnet/graph_BETA</PARAM>
21
				</PARAMETERS>
22
				<ARCS>
23
					<ARC to="rawGraph"/>
24
				</ARCS>
25
			</NODE>
26

  
27
			<NODE name="rawGraph" type="SubmitHadoopJob" isJoin="true">
28
				<DESCRIPTION>create the raw graph</DESCRIPTION>
29
				<PARAMETERS>
30
					<PARAM required="true" type="string" name="hadoopJob" managedBy="system">executeOozieJob</PARAM>
31
					<PARAM required="true" type="string" name="cluster"   managedBy="system">IIS</PARAM>
32
					<PARAM required="true" type="string" name="envParams" managedBy="system">
33
						{
34
						'graphBasePath' : 'graphBasePath'
35
						}
36
					</PARAM>
37
					<PARAM required="true" type="string" name="params" managedBy="system">
38
						{
39
						'oozie.wf.application.path' : '/lib/dnet/migration/wfs/regular_all_steps/oozie_app',
40
						'mongoURL' : 'mongodb://beta.services.openaire.eu',
41
						'mongoDb' : 'mdstore',
42
						'postgresURL' : 'jdbc:postgresql://beta.services.openaire.eu:5432/dnet_openaireplus',
43
						'postgresUser' : 'dnet',
44
						'postgresPassword' : 'dnetPwd',
45
						'workingPath' : '/tmp/dhp_migration_beta',
46
						'reuseContent' : 'true'
47
						}
48
					</PARAM>
49
					<PARAM required="true" type="boolean" name="simulation" managedBy="user">false</PARAM>
50
					<PARAM required="true" type="string" name="oozieReportActionsCsv" managedBy="system">build-report</PARAM>
51
				</PARAMETERS>
52
				<ARCS>
53
					<ARC to="success"/>
54
				</ARCS>
55
			</NODE>
56

  
57
		</CONFIGURATION>
58
		<STATUS>
59
		</STATUS>
60
	</BODY>
61
</RESOURCE_PROFILE>

Also available in: Unified diff