Project

General

Profile

1
<RESOURCE_PROFILE xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
2
    <HEADER>
3
        <RESOURCE_IDENTIFIER value="ba9a285a-f71a-4c53-b543-35ec06876f4e_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl"/>
4
        <RESOURCE_TYPE value="WorkflowDSResourceType"/>
5
        <RESOURCE_KIND value="WorkflowDSResources"/>
6
        <RESOURCE_URI value=""/>
7
        <DATE_OF_CREATION value="2017-07-04T18:39:17+02:00"/>
8
    </HEADER>
9
    <BODY>
10
        <WORKFLOW_NAME>Selective DB to HBase</WORKFLOW_NAME>
11
        <WORKFLOW_TYPE>Deduplication</WORKFLOW_TYPE>
12
        <WORKFLOW_PRIORITY>30</WORKFLOW_PRIORITY>
13
        <CONFIGURATION start="manual">
14
            <NODE isStart="true" name="start">
15
                <DESCRIPTION>start</DESCRIPTION>
16
                <PARAMETERS/>
17
                <ARCS>
18
                    <ARC to="SELECT_DS_ORG"/>
19
                    <ARC to="SELECT_DS"/>
20
                    <ARC to="SELECT_ORG"/>
21
                    <ARC to="SELECT_PER"/>
22
                    <ARC to="SELECT_CLAIM_REL"/>
23
                </ARCS>
24
            </NODE>
25
            <NODE name="SELECT_DS_ORG" type="Selection">
26
                <DESCRIPTION>Do we import the relationship datasource_organization?</DESCRIPTION>
27
                <PARAMETERS>
28
                    <PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">NO</PARAM>
29
                </PARAMETERS>
30
                <ARCS>
31
                    <ARC name="YES" to="queryDatasourceOrganization"/>
32
                    <ARC name="NO" to="doneDsOrgs"/>
33
                </ARCS>
34
            </NODE>
35
            <NODE name="queryDatasourceOrganization" type="QueryDb">
36
                <DESCRIPTION>query relation Datasource_Organization</DESCRIPTION>
37
                <PARAMETERS>
38
                    <PARAM managedBy="system" name="dbProperty" required="true" type="string">dnet.openaire.db.name</PARAM>
39
                    <PARAM managedBy="system" name="sql" required="true" type="string">/eu/dnetlib/msro/openaireplus/workflows/hbase/queryDatasourceOrganization.sql</PARAM>
40
                    <PARAM managedBy="system" name="outputEprParam" required="true" type="string">dsOrgEpr</PARAM>
41
                </PARAMETERS>
42
                <ARCS>
43
                    <ARC to="storeDatasourceOrganization"/>
44
                </ARCS>
45
            </NODE>
46
            <NODE name="SELECT_DS" type="Selection">
47
                <DESCRIPTION>Do we import the datasources?</DESCRIPTION>
48
                <PARAMETERS>
49
                    <PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">NO</PARAM>
50
                </PARAMETERS>
51
                <ARCS>
52
                    <ARC name="YES" to="prepareQueryDatasources"/>
53
                    <ARC name="NO" to="doneDatasources"/>
54
                </ARCS>
55
            </NODE>
56
            <NODE name="prepareQueryDatasources" type="ExecuteSql">
57
                <DESCRIPTION>query Datasources</DESCRIPTION>
58
                <PARAMETERS>
59
                    <PARAM managedBy="system" name="dbProperty" required="true" type="string">dnet.openaire.db.name</PARAM>
60
                    <PARAM managedBy="system" name="sql" required="true" type="string">/eu/dnetlib/msro/openaireplus/workflows/hbase/prepareQueryDatasources.sql</PARAM>
61
                </PARAMETERS>
62
                <ARCS>
63
                    <ARC to="queryDatasources"/>
64
                </ARCS>
65
            </NODE>
66
            <NODE name="queryDatasources" type="QueryDb">
67
                <DESCRIPTION>query Datasources</DESCRIPTION>
68
                <PARAMETERS>
69
                    <PARAM managedBy="system" name="dbProperty" required="true" type="string">dnet.openaire.db.name</PARAM>
70
                    <PARAM managedBy="system" name="sql" required="true" type="string">/eu/dnetlib/msro/openaireplus/workflows/hbase/queryDatasources.sql</PARAM>
71
                    <PARAM managedBy="system" name="sqlForSize" required="true" type="string">/eu/dnetlib/msro/openaireplus/workflows/hbase/queryDatasourcesForSize.sql</PARAM>
72
                    <PARAM managedBy="system" name="outputEprParam" required="true" type="string">dsEpr</PARAM>
73
                </PARAMETERS>
74
                <ARCS>
75
                    <ARC to="storeDatasources"/>
76
                </ARCS>
77
            </NODE>
78
            <NODE name="SELECT_ORG" type="Selection">
79
                <DESCRIPTION>Do we import the organizations?</DESCRIPTION>
80
                <PARAMETERS>
81
                    <PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">NO</PARAM>
82
                </PARAMETERS>
83
                <ARCS>
84
                    <ARC name="YES" to="queryOrganizations"/>
85
                    <ARC name="NO" to="doneOrganizations"/>
86
                </ARCS>
87
            </NODE>
88
            <NODE name="queryOrganizations" type="QueryDb">
89
                <DESCRIPTION>query Organizations</DESCRIPTION>
90
                <PARAMETERS>
91
                    <PARAM managedBy="system" name="dbProperty" required="true" type="string">dnet.openaire.db.name</PARAM>
92
                    <PARAM managedBy="system" name="sql" required="true" type="string">/eu/dnetlib/msro/openaireplus/workflows/hbase/queryOrganizations.sql</PARAM>
93
                    <PARAM managedBy="system" name="outputEprParam" required="true" type="string">orgEpr</PARAM>
94
                </PARAMETERS>
95
                <ARCS>
96
                    <ARC to="storeOrganizations"/>
97
                </ARCS>
98
            </NODE>
99
            <NODE name="SELECT_PER" type="Selection">
100
                <DESCRIPTION>Do we import the persons?</DESCRIPTION>
101
                <PARAMETERS>
102
                    <PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">NO</PARAM>
103
                </PARAMETERS>
104
                <ARCS>
105
                    <ARC name="YES" to="queryPersons"/>
106
                    <ARC name="NO" to="donePersons"/>
107
                </ARCS>
108
            </NODE>
109
            <NODE name="queryPersons" type="QueryDb">
110
                <DESCRIPTION>query Persons</DESCRIPTION>
111
                <PARAMETERS>
112
                    <PARAM managedBy="system" name="dbProperty" required="true" type="string">dnet.openaire.db.name</PARAM>
113
                    <PARAM managedBy="system" name="sql" required="true" type="string">/eu/dnetlib/msro/openaireplus/workflows/hbase/queryPersons.sql</PARAM>
114
                    <PARAM managedBy="system" name="outputEprParam" required="true" type="string">personsEpr</PARAM>
115
                </PARAMETERS>
116
                <ARCS>
117
                    <ARC to="storePersons"/>
118
                </ARCS>
119
            </NODE>
120
            <NODE name="SELECT_PRJ_ORG" type="Selection">
121
                <DESCRIPTION>Do we import the relationships project_organization?</DESCRIPTION>
122
                <PARAMETERS>
123
                    <PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">NO</PARAM>
124
                </PARAMETERS>
125
                <ARCS>
126
                    <ARC name="YES" to="queryProjectOrganization"/>
127
                    <ARC name="NO" to="donePrjOrgs"/>
128
                </ARCS>
129
            </NODE>
130
            <NODE name="queryProjectOrganization" type="QueryDb">
131
                <DESCRIPTION>query relation Project_Organization</DESCRIPTION>
132
                <PARAMETERS>
133
                    <PARAM managedBy="system" name="dbProperty" required="true" type="string">dnet.openaire.db.name</PARAM>
134
                    <PARAM managedBy="system" name="sql" required="true" type="string">/eu/dnetlib/msro/openaireplus/workflows/hbase/queryProjectOrganization.sql</PARAM>
135
                    <PARAM managedBy="system" name="outputEprParam" required="true" type="string">projOrgEpr</PARAM>
136
                </PARAMETERS>
137
                <ARCS>
138
                    <ARC to="storeProjectOrganization"/>
139
                </ARCS>
140
            </NODE>
141
            <NODE name="queryProjectContactperson" type="QueryDb">
142
                <DESCRIPTION>query relation Project_Contactperson</DESCRIPTION>
143
                <PARAMETERS>
144
                    <PARAM managedBy="system" name="dbProperty" required="true" type="string">dnet.openaire.db.name</PARAM>
145
                    <PARAM managedBy="system" name="sql" required="true" type="string">/eu/dnetlib/msro/openaireplus/workflows/hbase/queryProjectContactperson.sql</PARAM>
146
                    <PARAM managedBy="system" name="outputEprParam" required="true" type="string">projPerEpr</PARAM>
147
                </PARAMETERS>
148
                <ARCS>
149
                    <ARC to="storeProjectContactperson"/>
150
                </ARCS>
151
            </NODE>
152
            <NODE name="queryProjects" type="QueryDb">
153
                <DESCRIPTION>query Projects</DESCRIPTION>
154
                <PARAMETERS>
155
                    <PARAM managedBy="system" name="dbProperty" required="true" type="string">dnet.openaire.db.name</PARAM>
156
                    <PARAM managedBy="system" name="sql" required="true" type="string">/eu/dnetlib/msro/openaireplus/workflows/hbase/queryProjects.sql</PARAM>
157
                    <PARAM managedBy="system" name="outputEprParam" required="true" type="string">projEpr</PARAM>
158
                </PARAMETERS>
159
                <ARCS>
160
                    <ARC to="storeProjects"/>
161
                </ARCS>
162
            </NODE>
163
            <NODE name="SELECT_CLAIM_REL" type="Selection">
164
                <DESCRIPTION>Do we import the claimed relationships?</DESCRIPTION>
165
                <PARAMETERS>
166
                    <PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">YES</PARAM>
167
                </PARAMETERS>
168
                <ARCS>
169
                    <ARC name="YES" to="applyClaimRels"/>
170
                    <ARC name="NO" to="doneClaimRels"/>
171
                </ARCS>
172
            </NODE>
173
            <NODE name="applyClaimRels" type="ApplyClaimRels">
174
                <DESCRIPTION>Apply Claim Rels</DESCRIPTION>
175
                <PARAMETERS>
176
                    <PARAM managedBy="system" name="clusterName" required="true" type="string">DM</PARAM>
177
                    <PARAM managedBy="user" name="tableName" required="true" type="string">dev_openaire</PARAM>
178
                    <PARAM managedBy="user" name="sql" required="true" type="string">SELECT source_type, source_id, target_type, target_id, semantics FROM claim WHERE approved=TRUE AND source_type !='context'</PARAM>
179
                    <PARAM managedBy="user" name="countQuery" required="true" type="string">SELECT count(*) FROM claim WHERE approved=TRUE AND source_type !='context'</PARAM>
180
                </PARAMETERS>
181
                <ARCS>
182
                    <ARC to="doneClaimRels"/>
183
                </ARCS>
184
            </NODE>
185
            <NODE name="storeDatasourceOrganization" type="StoreHBase">
186
                <DESCRIPTION>Store relation Datasource_Organization</DESCRIPTION>
187
                <PARAMETERS>
188
                    <PARAM managedBy="system" name="inputEprParam" required="true" type="string">dsOrgEpr</PARAM>
189
                    <PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM>
190
                    <PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
191
                    <PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">760183e7-a5d4-4827-9218-3d49ff9fcbd1_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
192
                </PARAMETERS>
193
                <ARCS>
194
                    <ARC to="doneDsOrgs"/>
195
                </ARCS>
196
            </NODE>
197
            <NODE name="doneDsOrgs">
198
                <DESCRIPTION/>
199
                <PARAMETERS/>
200
                <ARCS>
201
                    <ARC to="checkpoint"/>
202
                </ARCS>
203
            </NODE>
204
            <NODE name="storeDatasources" type="StoreHBase">
205
                <DESCRIPTION>Store Datasources to HBase</DESCRIPTION>
206
                <PARAMETERS>
207
                    <PARAM managedBy="system" name="inputEprParam" required="true" type="string">dsEpr</PARAM>
208
                    <PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM>
209
                    <PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
210
                    <PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">30383440-fec9-4889-8ce8-147410d6e63c_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
211
                </PARAMETERS>
212
                <ARCS>
213
                    <ARC to="doneDatasources"/>
214
                </ARCS>
215
            </NODE>
216
            <NODE name="doneDatasources">
217
                <DESCRIPTION/>
218
                <PARAMETERS/>
219
                <ARCS>
220
                    <ARC to="checkpoint"/>
221
                </ARCS>
222
            </NODE>
223
            <NODE name="SELECT_PRJ_PER" type="Selection">
224
                <DESCRIPTION>Do we import the relationships project_contactperson?</DESCRIPTION>
225
                <PARAMETERS>
226
                    <PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">NO</PARAM>
227
                </PARAMETERS>
228
                <ARCS>
229
                    <ARC name="YES" to="queryProjectContactperson"/>
230
                    <ARC name="NO" to="donePrjPer"/>
231
                </ARCS>
232
            </NODE>
233
            <NODE name="storeOrganizations" type="StoreHBase">
234
                <DESCRIPTION>Store Organizations to HBase</DESCRIPTION>
235
                <PARAMETERS>
236
                    <PARAM managedBy="system" name="inputEprParam" required="true" type="string">orgEpr</PARAM>
237
                    <PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM>
238
                    <PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
239
                    <PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">4ef23346-95e3-4357-a4bf-e26d7c9c1b82_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
240
                </PARAMETERS>
241
                <ARCS>
242
                    <ARC to="doneOrganizations"/>
243
                </ARCS>
244
            </NODE>
245
            <NODE name="doneOrganizations">
246
                <DESCRIPTION/>
247
                <PARAMETERS/>
248
                <ARCS>
249
                    <ARC to="checkpoint"/>
250
                </ARCS>
251
            </NODE>
252
            <NODE name="storePersons" type="StoreHBase">
253
                <DESCRIPTION>Store Persons to HBase</DESCRIPTION>
254
                <PARAMETERS>
255
                    <PARAM managedBy="system" name="inputEprParam" required="true" type="string">personsEpr</PARAM>
256
                    <PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM>
257
                    <PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
258
                    <PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">dec9fb5d-c845-4ea3-a8a4-1b8c6b2f2682_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
259
                </PARAMETERS>
260
                <ARCS>
261
                    <ARC to="donePersons"/>
262
                </ARCS>
263
            </NODE>
264
            <NODE name="donePersons">
265
                <DESCRIPTION/>
266
                <PARAMETERS/>
267
                <ARCS>
268
                    <ARC to="checkpoint"/>
269
                </ARCS>
270
            </NODE>
271
            <NODE name="SELECT_PRJ" type="Selection">
272
                <DESCRIPTION>Do we import the projects?</DESCRIPTION>
273
                <PARAMETERS>
274
                    <PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">NO</PARAM>
275
                </PARAMETERS>
276
                <ARCS>
277
                    <ARC name="YES" to="queryProjects"/>
278
                    <ARC name="NO" to="doneProjects"/>
279
                </ARCS>
280
            </NODE>
281
            <NODE name="storeProjectOrganization" type="StoreHBase">
282
                <DESCRIPTION>Store relation Project_Organizations</DESCRIPTION>
283
                <PARAMETERS>
284
                    <PARAM managedBy="system" name="inputEprParam" required="true" type="string">projOrgEpr</PARAM>
285
                    <PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM>
286
                    <PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
287
                    <PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">621df4c6-ff71-4c1c-82bf-67a64c850be6_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
288
                </PARAMETERS>
289
                <ARCS>
290
                    <ARC to="donePrjOrgs"/>
291
                </ARCS>
292
            </NODE>
293
            <NODE name="donePrjOrgs">
294
                <DESCRIPTION/>
295
                <PARAMETERS/>
296
                <ARCS>
297
                    <ARC to="finish"/>
298
                </ARCS>
299
            </NODE>
300
            <NODE name="storeProjectContactperson" type="StoreHBase">
301
                <DESCRIPTION>Store relation Project_Contactperson</DESCRIPTION>
302
                <PARAMETERS>
303
                    <PARAM managedBy="system" name="inputEprParam" required="true" type="string">projPerEpr</PARAM>
304
                    <PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM>
305
                    <PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
306
                    <PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">9099673b-b46d-4aae-9f56-321607cbacba_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
307
                </PARAMETERS>
308
                <ARCS>
309
                    <ARC to="donePrjPer"/>
310
                </ARCS>
311
            </NODE>
312
            <NODE name="donePrjPer">
313
                <DESCRIPTION/>
314
                <PARAMETERS/>
315
                <ARCS>
316
                    <ARC to="finish"/>
317
                </ARCS>
318
            </NODE>
319
            <NODE name="storeProjects" type="StoreHBase">
320
                <DESCRIPTION>Store Projects to HBase</DESCRIPTION>
321
                <PARAMETERS>
322
                    <PARAM managedBy="system" name="inputEprParam" required="true" type="string">projEpr</PARAM>
323
                    <PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM>
324
                    <PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
325
                    <PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">5fccd61e-e6e5-4912-b849-54bc84b69a83_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
326
                </PARAMETERS>
327
                <ARCS>
328
                    <ARC to="doneProjects"/>
329
                </ARCS>
330
            </NODE>
331
            <NODE name="doneProjects">
332
                <DESCRIPTION/>
333
                <PARAMETERS/>
334
                <ARCS>
335
                    <ARC to="finish"/>
336
                </ARCS>
337
            </NODE>
338
            <NODE name="doneClaimRels">
339
                <DESCRIPTION/>
340
                <PARAMETERS/>
341
                <ARCS>
342
                    <ARC to="finish"/>
343
                </ARCS>
344
            </NODE>
345
            <NODE isJoin="true" name="checkpoint">
346
                <DESCRIPTION/>
347
                <PARAMETERS/>
348
                <ARCS>
349
                    <ARC to="SELECT_PRJ"/>
350
                    <ARC to="SELECT_CLAIM_REL"/>
351
                    <ARC to="SELECT_PRJ_PER"/>
352
                    <ARC to="SELECT_PRJ_ORG"/>
353
                </ARCS>
354
            </NODE>
355
            <NODE isJoin="true" name="finish">
356
                <DESCRIPTION/>
357
                <PARAMETERS/>
358
                <ARCS>
359
                    <ARC to="success"/>
360
                </ARCS>
361
            </NODE>
362
        </CONFIGURATION>
363
        <STATUS>
364
            <LAST_EXECUTION_ID>wf_20150409_131229_6</LAST_EXECUTION_ID>
365
            <LAST_EXECUTION_DATE>2015-04-09T13:13:10+02:00</LAST_EXECUTION_DATE>
366
            <LAST_EXECUTION_STATUS>SUCCESS</LAST_EXECUTION_STATUS>
367
            <LAST_EXECUTION_ERROR/>
368
            <LAST_EXECUTION_OUTPUT name="mainlog:storeOrganizations:count">33455</LAST_EXECUTION_OUTPUT>
369
        </STATUS>
370
    </BODY>
371
</RESOURCE_PROFILE>
(34-34/34)