Project

General

Profile

« Previous | Next » 

Revision 57579

Added by Alessia Bardi over 4 years ago

Fixes #4650: claims to enter HBASE before repository record

View differences:

modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/claims2hbase.xml
4 4 
		<RESOURCE_TYPE value="WorkflowDSResourceType"/>
5 5 
		<RESOURCE_KIND value="WorkflowDSResources"/>
6 6 
		<RESOURCE_URI value=""/>
7 
		<DATE_OF_CREATION value="2017-06-24T18:13:51.0Z"/>
7 
		<DATE_OF_CREATION value="2019-11-11T17:57:19+01:00"/>
8 8 
	</HEADER>
9 9 
	<BODY>
10 10 
		<WORKFLOW_NAME>Claim to HBase</WORKFLOW_NAME>
11 11 
		<WORKFLOW_TYPE>Data Load</WORKFLOW_TYPE>
12 12 
		<WORKFLOW_PRIORITY>30</WORKFLOW_PRIORITY>
13 13 
		<CONFIGURATION start="manual">
14 
			<NODE name="applyClaimRels" type="ApplyClaimRels" isStart="true">
14 
			<NODE isStart="true" name="setTable" type="SetHBaseTable">
15 
				<DESCRIPTION>set hbase table</DESCRIPTION>
16 
				<PARAMETERS>
17 
					<PARAM managedBy="user" name="table" required="true" type="string">dev_openaire</PARAM>
18 
				</PARAMETERS>
19 
				<ARCS>
20 
					<ARC to="mapreduceOAFClaim"/>
21 
					<ARC to="mapreduceODFClaim"/>
22 
				</ARCS>
23 
			</NODE>
24 
			<NODE isStart="true" name="prepareImportOAFClaims" type="PrepareMDStoreImport">
25 
				<DESCRIPTION>Configure export to HDFS for OAF claims</DESCRIPTION>
26 
				<PARAMETERS>
27 
					<PARAM managedBy="system" name="hdfsPathParam" required="true" type="string">hdfsPath</PARAM>
28 
					<PARAM managedBy="user" name="hdfsPath" required="true" type="string">/tmp/mdstores_oaf-store-claim_dev.seq</PARAM>
29 
					<PARAM managedBy="system" name="mappingParam" required="true" type="string">xslt</PARAM>
30 
					<PARAM function="obtainValues('oaf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">43108eba-4b67-4d7a-8928-55ca09f1c8ff_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
31 
				</PARAMETERS>
32 
				<ARCS>
33 
					<ARC to="exportOAFClaimRecords"/>
34 
				</ARCS>
35 
			</NODE>
36 
			<NODE name="exportOAFClaimRecords" type="MDStoreBatchExporter">
37 
				<DESCRIPTION>Fetch OAF claim records</DESCRIPTION>
38 
				<PARAMETERS>
39 
					<PARAM managedBy="user" name="format" required="true" type="string">OAF</PARAM>
40 
					<PARAM managedBy="user" name="layout" required="true" type="string">store</PARAM>
41 
					<PARAM managedBy="user" name="interpretation" required="true" type="string">claim</PARAM>
42 
					<PARAM managedBy="system" name="outputEprParam" required="true" type="string">records_epr</PARAM>
43 
				</PARAMETERS>
44 
				<ARCS>
45 
					<ARC to="storeOAFClaimRecords"/>
46 
				</ARCS>
47 
			</NODE>
48 
			<NODE name="storeOAFClaimRecords" type="StoreHdfsRecords">
49 
				<DESCRIPTION>Store OAF claims to HDFS</DESCRIPTION>
50 
				<PARAMETERS>
51 
					<PARAM managedBy="system" name="inputEprParam" required="true" type="string">records_epr</PARAM>
52 
					<PARAM managedBy="system" name="hdfsPathParam" required="true" type="string">hdfsPath</PARAM>
53 
					<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
54 
				</PARAMETERS>
55 
				<ARCS>
56 
					<ARC to="doneExportOAFClaim"/>
57 
				</ARCS>
58 
			</NODE>
59 
			<NODE name="doneExportOAFClaim">
60 
				<DESCRIPTION/>
61 
				<PARAMETERS/>
62 
				<ARCS>
63 
					<ARC to="mapreduceOAFClaim"/>
64 
				</ARCS>
65 
			</NODE>
66 
			<NODE isJoin="true" name="mapreduceOAFClaim" type="SubmitHadoopJob">
67 
				<DESCRIPTION>Run M/R import Job for OAF claims</DESCRIPTION>
68 
				<PARAMETERS>
69 
					<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
70 
					<PARAM managedBy="system" name="hadoopJob" required="true" type="string">mdStoreHdfsImportJob</PARAM>
71 
					<PARAM managedBy="user" name="simulation" required="true" type="boolean">false</PARAM>
72 
					<PARAM managedBy="system" name="envParams" required="true" type="string">
73 
						{
74 
						'mapred.input.dir' : 'hdfsPath',
75 
						'hbase.import.xslt' : 'xslt',
76 
						'hbase.mapred.outputtable' : 'hbaseTable'
77 
						}
78 
					</PARAM>
79 
				</PARAMETERS>
80 
				<ARCS>
81 
					<ARC to="goWithRelsAndUpdates"/>
82 
				</ARCS>
83 
			</NODE>
84 
			<NODE isStart="true" name="prepareImportODFClaims" type="PrepareMDStoreImport">
85 
				<DESCRIPTION>Configure export to HDFS for ODF claims</DESCRIPTION>
86 
				<PARAMETERS>
87 
					<PARAM managedBy="system" name="hdfsPathParam" required="true" type="string">hdfsPath</PARAM>
88 
					<PARAM managedBy="user" name="hdfsPath" required="true" type="string">/tmp/mdstores_odf-store-cleaned_dev.seq</PARAM>
89 
					<PARAM managedBy="system" name="mappingParam" required="true" type="string">xslt</PARAM>
90 
					<PARAM function="obtainValues('odf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">a8258f53-3cc0-4353-879f-699883fd1d0c_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
91 
				</PARAMETERS>
92 
				<ARCS>
93 
					<ARC to="exportODFClaimRecords"/>
94 
				</ARCS>
95 
			</NODE>
96 
			<NODE name="exportODFClaimRecords" type="MDStoreBatchExporter">
97 
				<DESCRIPTION>Fetch ODF claim records</DESCRIPTION>
98 
				<PARAMETERS>
99 
					<PARAM managedBy="user" name="format" required="true" type="string">ODF</PARAM>
100 
					<PARAM managedBy="user" name="layout" required="true" type="string">store</PARAM>
101 
					<PARAM managedBy="user" name="interpretation" required="true" type="string">claim</PARAM>
102 
					<PARAM managedBy="system" name="outputEprParam" required="true" type="string">records_epr</PARAM>
103 
				</PARAMETERS>
104 
				<ARCS>
105 
					<ARC to="storeODFClaimRecords"/>
106 
				</ARCS>
107 
			</NODE>
108 
			<NODE name="storeODFClaimRecords" type="StoreHdfsRecords">
109 
				<DESCRIPTION>Store records to HDFS</DESCRIPTION>
110 
				<PARAMETERS>
111 
					<PARAM managedBy="system" name="inputEprParam" required="true" type="string">records_epr</PARAM>
112 
					<PARAM managedBy="system" name="hdfsPathParam" required="true" type="string">hdfsPath</PARAM>
113 
					<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
114 
				</PARAMETERS>
115 
				<ARCS>
116 
					<ARC to="doneExportODFClaim"/>
117 
				</ARCS>
118 
			</NODE>
119 
			<NODE name="doneExportODFClaim">
120 
				<DESCRIPTION/>
121 
				<PARAMETERS/>
122 
				<ARCS>
123 
					<ARC to="mapreduceODFClaim"/>
124 
				</ARCS>
125 
			</NODE>
126 
			<NODE isJoin="true" name="mapreduceODFClaim" type="SubmitHadoopJob">
127 
				<DESCRIPTION>Run M/R import Job for ODF claims</DESCRIPTION>
128 
				<PARAMETERS>
129 
					<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
130 
					<PARAM managedBy="system" name="hadoopJob" required="true" type="string">mdStoreHdfsImportJob</PARAM>
131 
					<PARAM managedBy="user" name="simulation" required="true" type="boolean">false</PARAM>
132 
					<PARAM managedBy="system" name="envParams" required="true" type="string">
133 
						{
134 
						'mapred.input.dir' : 'hdfsPath',
135 
						'hbase.import.xslt' : 'xslt',
136 
						'hbase.mapred.outputtable' : 'hbaseTable'
137 
						}
138 
					</PARAM>
139 
				</PARAMETERS>
140 
				<ARCS>
141 
					<ARC to="goWithRelsAndUpdates"/>
142 
				</ARCS>
143 
			</NODE>
144 
			<NODE isJoin="true" name="goWithRelsAndUpdates">
145 
				<DESCRIPTION/>
146 
				<PARAMETERS/>
147 
				<ARCS>
148 
					<ARC to="applyClaimRels"/>
149 
				</ARCS>
150 
			</NODE>
151 
			<NODE name="applyClaimRels" type="ApplyClaimRels">
15 152 
				<DESCRIPTION>Apply Claim Rels</DESCRIPTION>
16 153 
				<PARAMETERS>
17 154 
					<PARAM managedBy="system" name="clusterName" required="true" type="string">DM</PARAM>
18 
					<PARAM managedBy="user" name="tableName" required="true" type="string"></PARAM>
155 
					<PARAM managedBy="user" name="tableName" required="true" type="string">dev_openaire</PARAM>
19 156 
					<PARAM managedBy="user" name="sql" required="true" type="string">SELECT source_type, source_id, target_type, target_id, semantics FROM claim WHERE approved=TRUE AND source_type !='context'</PARAM>
20 157 
					<PARAM managedBy="user" name="countQuery" required="true" type="string">SELECT count(*) FROM claim WHERE approved=TRUE AND source_type !='context'</PARAM>
21 158 
				</PARAMETERS>
......
27 164 
				<DESCRIPTION>Apply Claim Updates</DESCRIPTION>
28 165 
				<PARAMETERS>
29 166 
					<PARAM managedBy="system" name="clusterName" required="true" type="string">DM</PARAM>
30 
					<PARAM managedBy="user" name="tableName" required="true" type="string"></PARAM>
167 
					<PARAM managedBy="user" name="tableName" required="true" type="string">dev_openaire</PARAM>
31 168 
					<PARAM managedBy="user" name="sql" required="true" type="string">SELECT source_type, source_id, target_type, target_id, semantics FROM claim WHERE approved=TRUE AND source_type ='context'</PARAM>
32 169 
					<PARAM managedBy="user" name="countQuery" required="true" type="string">SELECT count(*) FROM claim WHERE approved=TRUE AND source_type ='context'</PARAM>
33 170 
				</PARAMETERS>
......
37 174 
			</NODE>
38 175 
		</CONFIGURATION>
39 176 
		<STATUS>
177 
			<LAST_EXECUTION_ID>wf_20191111_170912_747</LAST_EXECUTION_ID>
178 
			<LAST_EXECUTION_DATE>2019-11-11T17:36:30+01:00</LAST_EXECUTION_DATE>
179 
			<LAST_EXECUTION_STATUS>SUCCESS</LAST_EXECUTION_STATUS>
180 
			<LAST_EXECUTION_ERROR/>
40 181 
		</STATUS>
41 182 
	</BODY>
42 
</RESOURCE_PROFILE>
183 
</RESOURCE_PROFILE>
modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/dataRepository_claim_ingestion.xml
60 60 
				<DESCRIPTION>Create cleaned store</DESCRIPTION>
61 61 
				<PARAMETERS>
62 62 
					<PARAM name="format" managedBy="system" required="true" type="string">ODF</PARAM>
63 
					<PARAM name="interpretation" managedBy="system" required="true" type="string">cleaned</PARAM>
63 
					<PARAM name="interpretation" managedBy="system" required="true" type="string">claim</PARAM>
64 64 
					<PARAM name="layout" managedBy="system" required="true" type="string">store</PARAM>
65 65 
					<PARAM name="outputPrefix" managedBy="system" required="true" type="string">tran_</PARAM>
66 66 
				</PARAMETERS>
modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/pubsRepository_claim_ingestion.xml
60 60 
				<DESCRIPTION>Create cleaned store</DESCRIPTION>
61 61 
				<PARAMETERS>
62 62 
					<PARAM name="format" managedBy="system" required="true" type="string">OAF</PARAM>
63 
					<PARAM name="interpretation" managedBy="system" required="true" type="string">cleaned</PARAM>
63 
					<PARAM name="interpretation" managedBy="system" required="true" type="string">claim</PARAM>
64 64 
					<PARAM name="layout" managedBy="system" required="true" type="string">store</PARAM>
65 65 
					<PARAM name="outputPrefix" managedBy="system" required="true" type="string">tran_</PARAM>
66 66 
				</PARAMETERS>

Also available in: Unified diff