Project

General

Profile

1
<NODE name="SET_INFO" isStart="true" type="SetProviderInfo">
2
	<DESCRIPTION>Set information about current provider</DESCRIPTION>
3
	<PARAMETERS>
4
		<PARAM required="true" type="string" name="providerId" managedBy="system">$params.("dataprovider:id")$</PARAM>
5
		<PARAM required="true" type="string" name="providerName" managedBy="system">$params.("dataprovider:name")$</PARAM>
6
		<PARAM required="true" type="string" name="api" managedBy="system">$params.("dataprovider:interface")$</PARAM>
7
	</PARAMETERS>
8
	<ARCS>
9
		<ARC to="obtainParams"/>
10
	</ARCS>
11
</NODE>
12

    
13
<NODE name="obtainParams" type="ObtainOpenaireDataSourceParams">
14
	<DESCRIPTION>Obtain data source params</DESCRIPTION>
15
	<PARAMETERS>
16
		<PARAM required="true" type="string" name="providerId" managedBy="system">$params.("dataprovider:id")$</PARAM>
17
	</PARAMETERS>
18
	<ARCS>
19
		<ARC to="PREPARE_ENV_COLLECTION"/>
20
	</ARCS>
21
</NODE>
22

    
23
<NODE name="PREPARE_ENV_COLLECTION" type="PrepareEnvCollectHadoopJobNode">
24
    <DESCRIPTION>Set in the environment all the variable needed to the collection oozie job</DESCRIPTION>
25
    <PARAMETERS>
26
        <PARAM required="true" type="string" name="mdId" managedBy="system" category="MDSTORE_ID">$params.("harv_id")$</PARAM>
27
        <PARAM required="true" type="string" name="collectionMode" managedBy="user" function="validValues(['REFRESH','INCREMENTAL'])"></PARAM>
28
        <PARAM required="true" type="string" name="metadataEncoding" managedBy="user">XML</PARAM>
29
        <PARAM managedBy="user" name="fromDateOverride" required="false" type="string"/>
30
        <PARAM managedBy="user" name="maxNumberOfRetry" required="false" type="int">5</PARAM>
31
        <PARAM managedBy="user" name="requestDelay" required="false" type="int">0</PARAM>
32
        <PARAM managedBy="user" name="retryDelay" required="false" type="int">60</PARAM>
33
        <PARAM managedBy="user" name="connectTimeOut" required="false" type="int">30</PARAM>
34
        <PARAM managedBy="user" name="readTimeOut" required="false" type="int">60</PARAM>
35
    </PARAMETERS>
36
    <ARCS>
37
        <ARC to="COLLECT_HADOOP"/>
38
    </ARCS>
39
</NODE>
40

    
41
<NODE name="COLLECT_HADOOP" type="SubmitDnetHadoopJobNode">
42
    <DESCRIPTION>Start the Hadoop Job</DESCRIPTION>
43
    <PARAMETERS>
44
        <PARAM managedBy="system" name="hadoopJob" required="true" type="string">executeOozieJob</PARAM>
45
        <PARAM managedBy="user" name="cluster" required="true" type="string">DHP</PARAM>
46
        <PARAM managedBy="system" name="envParams" required="true" type="string">
47
            {
48
                "apiDescription":"apiDescription",
49
                "dataSourceInfo":"dataSourceInfo",
50
                "identifierPath":"identifierPath",
51
                "metadataEncoding":"metadataEncoding",
52
                "timestamp":"timestamp",
53
                "workflowId":"workflowId",
54
                "mdStoreID":"mdId",
55
                "collectionMode":"collectionMode",
56
                "maxNumberOfRetry":"maxNumberOfRetry",
57
                "requestDelay":"requestDelay",
58
                "retryDelay":"retryDelay",
59
                "connectTimeOut":"connectTimeOut",
60
                "readTimeOut":"readTimeOut",
61
                "dnetMessageManagerURL":"dnetMessageManagerURL",
62
                "oozie.wf.application.path":"oozieWfPath"
63
            }
64
        </PARAM>
65
        <PARAM managedBy="system" name="params" required="true" type="string">
66
            {
67
                "collection_java_xmx" : "-Xmx300m"
68
            }
69
        </PARAM
70
        <PARAM managedBy="system" name="oozieReportActionsCsv" required="true" type="string">BeginRead,StartTransaction,CollectionWorker</PARAM>
71
    </PARAMETERS>
72
    <ARCS>
73
        <ARC to="UPDATE_INFO"/>
74
    </ARCS>
75
</NODE>
76

    
77
<NODE name="UPDATE_INFO" type="MDStoreToApiExtraFieldHadoop">
78
	<DESCRIPTION>Update datasouce API extra fields</DESCRIPTION>
79
	<PARAMETERS>
80
		<PARAM required="true" type="string" name="mdId" managedBy="system">$params.("harv_id")$</PARAM>
81
		<PARAM required="true" type="string" name="datasourceId" managedBy="system">$params.("dataprovider:id")$</PARAM>
82
		<PARAM required="true" type="string" name="datasourceInterface" managedBy="system">$params.("dataprovider:interface")$</PARAM>
83
		<PARAM required="true" type="string" name="extraFieldForTotal" managedBy="system">last_collection_total</PARAM>
84
		<PARAM required="true" type="string" name="extraFieldForDate" managedBy="system">last_collection_date</PARAM>
85
		<PARAM required="true" type="string" name="extraFieldForMdId" managedBy="system">last_collection_mdId</PARAM>
86
	</PARAMETERS>
87
	<ARCS>
88
		<ARC to="success"/>
89
	</ARCS>
90
</NODE>
(1-1/3)