Revision 57579
Added by Alessia Bardi over 4 years ago
claims2hbase.xml | ||
---|---|---|
4 | 4 |
<RESOURCE_TYPE value="WorkflowDSResourceType"/> |
5 | 5 |
<RESOURCE_KIND value="WorkflowDSResources"/> |
6 | 6 |
<RESOURCE_URI value=""/> |
7 |
<DATE_OF_CREATION value="2017-06-24T18:13:51.0Z"/>
|
|
7 |
<DATE_OF_CREATION value="2019-11-11T17:57:19+01:00"/>
|
|
8 | 8 |
</HEADER> |
9 | 9 |
<BODY> |
10 | 10 |
<WORKFLOW_NAME>Claim to HBase</WORKFLOW_NAME> |
11 | 11 |
<WORKFLOW_TYPE>Data Load</WORKFLOW_TYPE> |
12 | 12 |
<WORKFLOW_PRIORITY>30</WORKFLOW_PRIORITY> |
13 | 13 |
<CONFIGURATION start="manual"> |
14 |
<NODE name="applyClaimRels" type="ApplyClaimRels" isStart="true"> |
|
14 |
<NODE isStart="true" name="setTable" type="SetHBaseTable"> |
|
15 |
<DESCRIPTION>set hbase table</DESCRIPTION> |
|
16 |
<PARAMETERS> |
|
17 |
<PARAM managedBy="user" name="table" required="true" type="string">dev_openaire</PARAM> |
|
18 |
</PARAMETERS> |
|
19 |
<ARCS> |
|
20 |
<ARC to="mapreduceOAFClaim"/> |
|
21 |
<ARC to="mapreduceODFClaim"/> |
|
22 |
</ARCS> |
|
23 |
</NODE> |
|
24 |
<NODE isStart="true" name="prepareImportOAFClaims" type="PrepareMDStoreImport"> |
|
25 |
<DESCRIPTION>Configure export to HDFS for OAF claims</DESCRIPTION> |
|
26 |
<PARAMETERS> |
|
27 |
<PARAM managedBy="system" name="hdfsPathParam" required="true" type="string">hdfsPath</PARAM> |
|
28 |
<PARAM managedBy="user" name="hdfsPath" required="true" type="string">/tmp/mdstores_oaf-store-claim_dev.seq</PARAM> |
|
29 |
<PARAM managedBy="system" name="mappingParam" required="true" type="string">xslt</PARAM> |
|
30 |
<PARAM function="obtainValues('oaf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">43108eba-4b67-4d7a-8928-55ca09f1c8ff_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM> |
|
31 |
</PARAMETERS> |
|
32 |
<ARCS> |
|
33 |
<ARC to="exportOAFClaimRecords"/> |
|
34 |
</ARCS> |
|
35 |
</NODE> |
|
36 |
<NODE name="exportOAFClaimRecords" type="MDStoreBatchExporter"> |
|
37 |
<DESCRIPTION>Fetch OAF claim records</DESCRIPTION> |
|
38 |
<PARAMETERS> |
|
39 |
<PARAM managedBy="user" name="format" required="true" type="string">OAF</PARAM> |
|
40 |
<PARAM managedBy="user" name="layout" required="true" type="string">store</PARAM> |
|
41 |
<PARAM managedBy="user" name="interpretation" required="true" type="string">claim</PARAM> |
|
42 |
<PARAM managedBy="system" name="outputEprParam" required="true" type="string">records_epr</PARAM> |
|
43 |
</PARAMETERS> |
|
44 |
<ARCS> |
|
45 |
<ARC to="storeOAFClaimRecords"/> |
|
46 |
</ARCS> |
|
47 |
</NODE> |
|
48 |
<NODE name="storeOAFClaimRecords" type="StoreHdfsRecords"> |
|
49 |
<DESCRIPTION>Store OAF claims to HDFS</DESCRIPTION> |
|
50 |
<PARAMETERS> |
|
51 |
<PARAM managedBy="system" name="inputEprParam" required="true" type="string">records_epr</PARAM> |
|
52 |
<PARAM managedBy="system" name="hdfsPathParam" required="true" type="string">hdfsPath</PARAM> |
|
53 |
<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM> |
|
54 |
</PARAMETERS> |
|
55 |
<ARCS> |
|
56 |
<ARC to="doneExportOAFClaim"/> |
|
57 |
</ARCS> |
|
58 |
</NODE> |
|
59 |
<NODE name="doneExportOAFClaim"> |
|
60 |
<DESCRIPTION/> |
|
61 |
<PARAMETERS/> |
|
62 |
<ARCS> |
|
63 |
<ARC to="mapreduceOAFClaim"/> |
|
64 |
</ARCS> |
|
65 |
</NODE> |
|
66 |
<NODE isJoin="true" name="mapreduceOAFClaim" type="SubmitHadoopJob"> |
|
67 |
<DESCRIPTION>Run M/R import Job for OAF claims</DESCRIPTION> |
|
68 |
<PARAMETERS> |
|
69 |
<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM> |
|
70 |
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">mdStoreHdfsImportJob</PARAM> |
|
71 |
<PARAM managedBy="user" name="simulation" required="true" type="boolean">false</PARAM> |
|
72 |
<PARAM managedBy="system" name="envParams" required="true" type="string"> |
|
73 |
{ |
|
74 |
'mapred.input.dir' : 'hdfsPath', |
|
75 |
'hbase.import.xslt' : 'xslt', |
|
76 |
'hbase.mapred.outputtable' : 'hbaseTable' |
|
77 |
} |
|
78 |
</PARAM> |
|
79 |
</PARAMETERS> |
|
80 |
<ARCS> |
|
81 |
<ARC to="goWithRelsAndUpdates"/> |
|
82 |
</ARCS> |
|
83 |
</NODE> |
|
84 |
<NODE isStart="true" name="prepareImportODFClaims" type="PrepareMDStoreImport"> |
|
85 |
<DESCRIPTION>Configure export to HDFS for ODF claims</DESCRIPTION> |
|
86 |
<PARAMETERS> |
|
87 |
<PARAM managedBy="system" name="hdfsPathParam" required="true" type="string">hdfsPath</PARAM> |
|
88 |
<PARAM managedBy="user" name="hdfsPath" required="true" type="string">/tmp/mdstores_odf-store-cleaned_dev.seq</PARAM> |
|
89 |
<PARAM managedBy="system" name="mappingParam" required="true" type="string">xslt</PARAM> |
|
90 |
<PARAM function="obtainValues('odf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">a8258f53-3cc0-4353-879f-699883fd1d0c_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM> |
|
91 |
</PARAMETERS> |
|
92 |
<ARCS> |
|
93 |
<ARC to="exportODFClaimRecords"/> |
|
94 |
</ARCS> |
|
95 |
</NODE> |
|
96 |
<NODE name="exportODFClaimRecords" type="MDStoreBatchExporter"> |
|
97 |
<DESCRIPTION>Fetch ODF claim records</DESCRIPTION> |
|
98 |
<PARAMETERS> |
|
99 |
<PARAM managedBy="user" name="format" required="true" type="string">ODF</PARAM> |
|
100 |
<PARAM managedBy="user" name="layout" required="true" type="string">store</PARAM> |
|
101 |
<PARAM managedBy="user" name="interpretation" required="true" type="string">claim</PARAM> |
|
102 |
<PARAM managedBy="system" name="outputEprParam" required="true" type="string">records_epr</PARAM> |
|
103 |
</PARAMETERS> |
|
104 |
<ARCS> |
|
105 |
<ARC to="storeODFClaimRecords"/> |
|
106 |
</ARCS> |
|
107 |
</NODE> |
|
108 |
<NODE name="storeODFClaimRecords" type="StoreHdfsRecords"> |
|
109 |
<DESCRIPTION>Store records to HDFS</DESCRIPTION> |
|
110 |
<PARAMETERS> |
|
111 |
<PARAM managedBy="system" name="inputEprParam" required="true" type="string">records_epr</PARAM> |
|
112 |
<PARAM managedBy="system" name="hdfsPathParam" required="true" type="string">hdfsPath</PARAM> |
|
113 |
<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM> |
|
114 |
</PARAMETERS> |
|
115 |
<ARCS> |
|
116 |
<ARC to="doneExportODFClaim"/> |
|
117 |
</ARCS> |
|
118 |
</NODE> |
|
119 |
<NODE name="doneExportODFClaim"> |
|
120 |
<DESCRIPTION/> |
|
121 |
<PARAMETERS/> |
|
122 |
<ARCS> |
|
123 |
<ARC to="mapreduceODFClaim"/> |
|
124 |
</ARCS> |
|
125 |
</NODE> |
|
126 |
<NODE isJoin="true" name="mapreduceODFClaim" type="SubmitHadoopJob"> |
|
127 |
<DESCRIPTION>Run M/R import Job for ODF claims</DESCRIPTION> |
|
128 |
<PARAMETERS> |
|
129 |
<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM> |
|
130 |
<PARAM managedBy="system" name="hadoopJob" required="true" type="string">mdStoreHdfsImportJob</PARAM> |
|
131 |
<PARAM managedBy="user" name="simulation" required="true" type="boolean">false</PARAM> |
|
132 |
<PARAM managedBy="system" name="envParams" required="true" type="string"> |
|
133 |
{ |
|
134 |
'mapred.input.dir' : 'hdfsPath', |
|
135 |
'hbase.import.xslt' : 'xslt', |
|
136 |
'hbase.mapred.outputtable' : 'hbaseTable' |
|
137 |
} |
|
138 |
</PARAM> |
|
139 |
</PARAMETERS> |
|
140 |
<ARCS> |
|
141 |
<ARC to="goWithRelsAndUpdates"/> |
|
142 |
</ARCS> |
|
143 |
</NODE> |
|
144 |
<NODE isJoin="true" name="goWithRelsAndUpdates"> |
|
145 |
<DESCRIPTION/> |
|
146 |
<PARAMETERS/> |
|
147 |
<ARCS> |
|
148 |
<ARC to="applyClaimRels"/> |
|
149 |
</ARCS> |
|
150 |
</NODE> |
|
151 |
<NODE name="applyClaimRels" type="ApplyClaimRels"> |
|
15 | 152 |
<DESCRIPTION>Apply Claim Rels</DESCRIPTION> |
16 | 153 |
<PARAMETERS> |
17 | 154 |
<PARAM managedBy="system" name="clusterName" required="true" type="string">DM</PARAM> |
18 |
<PARAM managedBy="user" name="tableName" required="true" type="string"></PARAM> |
|
155 |
<PARAM managedBy="user" name="tableName" required="true" type="string">dev_openaire</PARAM>
|
|
19 | 156 |
<PARAM managedBy="user" name="sql" required="true" type="string">SELECT source_type, source_id, target_type, target_id, semantics FROM claim WHERE approved=TRUE AND source_type !='context'</PARAM> |
20 | 157 |
<PARAM managedBy="user" name="countQuery" required="true" type="string">SELECT count(*) FROM claim WHERE approved=TRUE AND source_type !='context'</PARAM> |
21 | 158 |
</PARAMETERS> |
... | ... | |
27 | 164 |
<DESCRIPTION>Apply Claim Updates</DESCRIPTION> |
28 | 165 |
<PARAMETERS> |
29 | 166 |
<PARAM managedBy="system" name="clusterName" required="true" type="string">DM</PARAM> |
30 |
<PARAM managedBy="user" name="tableName" required="true" type="string"></PARAM> |
|
167 |
<PARAM managedBy="user" name="tableName" required="true" type="string">dev_openaire</PARAM>
|
|
31 | 168 |
<PARAM managedBy="user" name="sql" required="true" type="string">SELECT source_type, source_id, target_type, target_id, semantics FROM claim WHERE approved=TRUE AND source_type ='context'</PARAM> |
32 | 169 |
<PARAM managedBy="user" name="countQuery" required="true" type="string">SELECT count(*) FROM claim WHERE approved=TRUE AND source_type ='context'</PARAM> |
33 | 170 |
</PARAMETERS> |
... | ... | |
37 | 174 |
</NODE> |
38 | 175 |
</CONFIGURATION> |
39 | 176 |
<STATUS> |
177 |
<LAST_EXECUTION_ID>wf_20191111_170912_747</LAST_EXECUTION_ID> |
|
178 |
<LAST_EXECUTION_DATE>2019-11-11T17:36:30+01:00</LAST_EXECUTION_DATE> |
|
179 |
<LAST_EXECUTION_STATUS>SUCCESS</LAST_EXECUTION_STATUS> |
|
180 |
<LAST_EXECUTION_ERROR/> |
|
40 | 181 |
</STATUS> |
41 | 182 |
</BODY> |
42 |
</RESOURCE_PROFILE> |
|
183 |
</RESOURCE_PROFILE> |
Also available in: Unified diff
Fixes #4650: claims to enter HBASE before repository record