Revision 48331
Added by Alessia Bardi almost 7 years ago
modules/dnet-deduplication/trunk/src/main/resources/eu/dnetlib/test/profiles/meta/workflows/selective.db2hbase.xml | ||
---|---|---|
4 | 4 |
<RESOURCE_TYPE value="WorkflowDSResourceType"/> |
5 | 5 |
<RESOURCE_KIND value="WorkflowDSResources"/> |
6 | 6 |
<RESOURCE_URI value=""/> |
7 |
<DATE_OF_CREATION value="2015-02-18T10:47:40+01:00"/>
|
|
7 |
<DATE_OF_CREATION value="2017-07-04T18:39:17+02:00"/>
|
|
8 | 8 |
</HEADER> |
9 | 9 |
<BODY> |
10 | 10 |
<WORKFLOW_NAME>Selective DB to HBase</WORKFLOW_NAME> |
... | ... | |
19 | 19 |
<ARC to="SELECT_DS"/> |
20 | 20 |
<ARC to="SELECT_ORG"/> |
21 | 21 |
<ARC to="SELECT_PER"/> |
22 |
<ARC to="SELECT_CLAIM_REL"/> |
|
22 | 23 |
</ARCS> |
23 | 24 |
</NODE> |
24 | 25 |
<NODE name="SELECT_DS_ORG" type="Selection"> |
... | ... | |
77 | 78 |
<NODE name="SELECT_ORG" type="Selection"> |
78 | 79 |
<DESCRIPTION>Do we import the organizations?</DESCRIPTION> |
79 | 80 |
<PARAMETERS> |
80 |
<PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">YES</PARAM>
|
|
81 |
<PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">NO</PARAM>
|
|
81 | 82 |
</PARAMETERS> |
82 | 83 |
<ARCS> |
83 | 84 |
<ARC name="YES" to="queryOrganizations"/> |
... | ... | |
159 | 160 |
<ARC to="storeProjects"/> |
160 | 161 |
</ARCS> |
161 | 162 |
</NODE> |
162 |
<NODE name="queryClaimRels" type="QueryDb">
|
|
163 |
<DESCRIPTION>query Claim Rels</DESCRIPTION>
|
|
163 |
<NODE name="SELECT_CLAIM_REL" type="Selection">
|
|
164 |
<DESCRIPTION>Do we import the claimed relationships?</DESCRIPTION>
|
|
164 | 165 |
<PARAMETERS> |
165 |
<PARAM managedBy="system" name="dbProperty" required="true" type="string">dnet.openaire.db.name</PARAM> |
|
166 |
<PARAM managedBy="system" name="sql" required="true" type="string">/eu/dnetlib/msro/openaireplus/workflows/hbase/queryClaimsRel.sql</PARAM> |
|
167 |
<PARAM managedBy="system" name="outputEprParam" required="true" type="string">claimRelsEpr</PARAM> |
|
168 |
<PARAM managedBy="system" name="xslt" required="true" type="string">/eu/dnetlib/msro/openaireplus/workflows/db/decapsule_claims_rels.xsl</PARAM> |
|
166 |
<PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">YES</PARAM> |
|
169 | 167 |
</PARAMETERS> |
170 | 168 |
<ARCS> |
171 |
<ARC to="storeClaimRels"/> |
|
169 |
<ARC name="YES" to="applyClaimRels"/> |
|
170 |
<ARC name="NO" to="doneClaimRels"/> |
|
172 | 171 |
</ARCS> |
173 | 172 |
</NODE> |
173 |
<NODE name="applyClaimRels" type="ApplyClaimRels"> |
|
174 |
<DESCRIPTION>Apply Claim Rels</DESCRIPTION> |
|
175 |
<PARAMETERS> |
|
176 |
<PARAM managedBy="system" name="clusterName" required="true" type="string">DM</PARAM> |
|
177 |
<PARAM managedBy="user" name="tableName" required="true" type="string">dev_openaire</PARAM> |
|
178 |
<PARAM managedBy="user" name="sql" required="true" type="string">SELECT source_type, source_id, target_type, target_id, semantics FROM claim WHERE approved=TRUE AND source_type !='context'</PARAM> |
|
179 |
<PARAM managedBy="user" name="countQuery" required="true" type="string">SELECT count(*) FROM claim WHERE approved=TRUE AND source_type !='context'</PARAM> |
|
180 |
</PARAMETERS> |
|
181 |
<ARCS> |
|
182 |
<ARC to="doneClaimRels"/> |
|
183 |
</ARCS> |
|
184 |
</NODE> |
|
174 | 185 |
<NODE name="storeDatasourceOrganization" type="StoreHBase"> |
175 | 186 |
<DESCRIPTION>Store relation Datasource_Organization</DESCRIPTION> |
176 | 187 |
<PARAMETERS> |
177 | 188 |
<PARAM managedBy="system" name="inputEprParam" required="true" type="string">dsOrgEpr</PARAM> |
178 | 189 |
<PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM> |
179 | 190 |
<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM> |
180 |
<PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM>
|
|
191 |
<PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">760183e7-a5d4-4827-9218-3d49ff9fcbd1_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
|
|
181 | 192 |
</PARAMETERS> |
182 | 193 |
<ARCS> |
183 | 194 |
<ARC to="doneDsOrgs"/> |
... | ... | |
196 | 207 |
<PARAM managedBy="system" name="inputEprParam" required="true" type="string">dsEpr</PARAM> |
197 | 208 |
<PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM> |
198 | 209 |
<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM> |
199 |
<PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM>
|
|
210 |
<PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">30383440-fec9-4889-8ce8-147410d6e63c_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
|
|
200 | 211 |
</PARAMETERS> |
201 | 212 |
<ARCS> |
202 | 213 |
<ARC to="doneDatasources"/> |
... | ... | |
225 | 236 |
<PARAM managedBy="system" name="inputEprParam" required="true" type="string">orgEpr</PARAM> |
226 | 237 |
<PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM> |
227 | 238 |
<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM> |
228 |
<PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM>
|
|
239 |
<PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">4ef23346-95e3-4357-a4bf-e26d7c9c1b82_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
|
|
229 | 240 |
</PARAMETERS> |
230 | 241 |
<ARCS> |
231 | 242 |
<ARC to="doneOrganizations"/> |
... | ... | |
238 | 249 |
<ARC to="checkpoint"/> |
239 | 250 |
</ARCS> |
240 | 251 |
</NODE> |
241 |
<NODE name="SELECT_CLAIM_REL" type="Selection"> |
|
242 |
<DESCRIPTION>Do we import the claimed relationships?</DESCRIPTION> |
|
243 |
<PARAMETERS> |
|
244 |
<PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">NO</PARAM> |
|
245 |
</PARAMETERS> |
|
246 |
<ARCS> |
|
247 |
<ARC name="YES" to="queryClaimRels"/> |
|
248 |
<ARC name="NO" to="doneClaimRels"/> |
|
249 |
</ARCS> |
|
250 |
</NODE> |
|
251 | 252 |
<NODE name="storePersons" type="StoreHBase"> |
252 | 253 |
<DESCRIPTION>Store Persons to HBase</DESCRIPTION> |
253 | 254 |
<PARAMETERS> |
254 | 255 |
<PARAM managedBy="system" name="inputEprParam" required="true" type="string">personsEpr</PARAM> |
255 | 256 |
<PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM> |
256 | 257 |
<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM> |
257 |
<PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM>
|
|
258 |
<PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">dec9fb5d-c845-4ea3-a8a4-1b8c6b2f2682_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
|
|
258 | 259 |
</PARAMETERS> |
259 | 260 |
<ARCS> |
260 | 261 |
<ARC to="donePersons"/> |
... | ... | |
283 | 284 |
<PARAM managedBy="system" name="inputEprParam" required="true" type="string">projOrgEpr</PARAM> |
284 | 285 |
<PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM> |
285 | 286 |
<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM> |
286 |
<PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM>
|
|
287 |
<PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">621df4c6-ff71-4c1c-82bf-67a64c850be6_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
|
|
287 | 288 |
</PARAMETERS> |
288 | 289 |
<ARCS> |
289 | 290 |
<ARC to="donePrjOrgs"/> |
... | ... | |
302 | 303 |
<PARAM managedBy="system" name="inputEprParam" required="true" type="string">projPerEpr</PARAM> |
303 | 304 |
<PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM> |
304 | 305 |
<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM> |
305 |
<PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM>
|
|
306 |
<PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">9099673b-b46d-4aae-9f56-321607cbacba_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
|
|
306 | 307 |
</PARAMETERS> |
307 | 308 |
<ARCS> |
308 | 309 |
<ARC to="donePrjPer"/> |
... | ... | |
321 | 322 |
<PARAM managedBy="system" name="inputEprParam" required="true" type="string">projEpr</PARAM> |
322 | 323 |
<PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM> |
323 | 324 |
<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM> |
324 |
<PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM>
|
|
325 |
<PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">5fccd61e-e6e5-4912-b849-54bc84b69a83_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
|
|
325 | 326 |
</PARAMETERS> |
326 | 327 |
<ARCS> |
327 | 328 |
<ARC to="doneProjects"/> |
... | ... | |
334 | 335 |
<ARC to="finish"/> |
335 | 336 |
</ARCS> |
336 | 337 |
</NODE> |
337 |
<NODE name="storeClaimRels" type="StoreHBase"> |
|
338 |
<DESCRIPTION>Store relation Claim Project</DESCRIPTION> |
|
339 |
<PARAMETERS> |
|
340 |
<PARAM managedBy="system" name="inputEprParam" required="true" type="string">claimRelsEpr</PARAM> |
|
341 |
<PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM> |
|
342 |
<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM> |
|
343 |
<PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM> |
|
344 |
</PARAMETERS> |
|
345 |
<ARCS> |
|
346 |
<ARC to="doneClaimRels"/> |
|
347 |
</ARCS> |
|
348 |
</NODE> |
|
349 | 338 |
<NODE name="doneClaimRels"> |
350 | 339 |
<DESCRIPTION/> |
351 | 340 |
<PARAMETERS/> |
... | ... | |
379 | 368 |
<LAST_EXECUTION_OUTPUT name="mainlog:storeOrganizations:count">33455</LAST_EXECUTION_OUTPUT> |
380 | 369 |
</STATUS> |
381 | 370 |
</BODY> |
382 |
</RESOURCE_PROFILE> |
|
371 |
</RESOURCE_PROFILE> |
Also available in: Unified diff
updated workflow for the claim part