Project

General

Profile

« Previous | Next » 

Revision 48331

Added by Alessia Bardi almost 7 years ago

updated workflow for the claim part

View differences:

selective.db2hbase.xml
4 4 
        <RESOURCE_TYPE value="WorkflowDSResourceType"/>
5 5 
        <RESOURCE_KIND value="WorkflowDSResources"/>
6 6 
        <RESOURCE_URI value=""/>
7 
        <DATE_OF_CREATION value="2015-02-18T10:47:40+01:00"/>
7 
        <DATE_OF_CREATION value="2017-07-04T18:39:17+02:00"/>
8 8 
    </HEADER>
9 9 
    <BODY>
10 10 
        <WORKFLOW_NAME>Selective DB to HBase</WORKFLOW_NAME>
......
19 19 
                    <ARC to="SELECT_DS"/>
20 20 
                    <ARC to="SELECT_ORG"/>
21 21 
                    <ARC to="SELECT_PER"/>
22 
                    <ARC to="SELECT_CLAIM_REL"/>
22 23 
                </ARCS>
23 24 
            </NODE>
24 25 
            <NODE name="SELECT_DS_ORG" type="Selection">
......
77 78 
            <NODE name="SELECT_ORG" type="Selection">
78 79 
                <DESCRIPTION>Do we import the organizations?</DESCRIPTION>
79 80 
                <PARAMETERS>
80 
                    <PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">YES</PARAM>
81 
                    <PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">NO</PARAM>
81 82 
                </PARAMETERS>
82 83 
                <ARCS>
83 84 
                    <ARC name="YES" to="queryOrganizations"/>
......
159 160 
                    <ARC to="storeProjects"/>
160 161 
                </ARCS>
161 162 
            </NODE>
162 
            <NODE name="queryClaimRels" type="QueryDb">
163 
                <DESCRIPTION>query Claim Rels</DESCRIPTION>
163 
            <NODE name="SELECT_CLAIM_REL" type="Selection">
164 
                <DESCRIPTION>Do we import the claimed relationships?</DESCRIPTION>
164 165 
                <PARAMETERS>
165 
                    <PARAM managedBy="system" name="dbProperty" required="true" type="string">dnet.openaire.db.name</PARAM>
166 
                    <PARAM managedBy="system" name="sql" required="true" type="string">/eu/dnetlib/msro/openaireplus/workflows/hbase/queryClaimsRel.sql</PARAM>
167 
                    <PARAM managedBy="system" name="outputEprParam" required="true" type="string">claimRelsEpr</PARAM>
168 
                    <PARAM managedBy="system" name="xslt" required="true" type="string">/eu/dnetlib/msro/openaireplus/workflows/db/decapsule_claims_rels.xsl</PARAM>
166 
                    <PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">YES</PARAM>
169 167 
                </PARAMETERS>
170 168 
                <ARCS>
171 
                    <ARC to="storeClaimRels"/>
169 
                    <ARC name="YES" to="applyClaimRels"/>
170 
                    <ARC name="NO" to="doneClaimRels"/>
172 171 
                </ARCS>
173 172 
            </NODE>
173 
            <NODE name="applyClaimRels" type="ApplyClaimRels">
174 
                <DESCRIPTION>Apply Claim Rels</DESCRIPTION>
175 
                <PARAMETERS>
176 
                    <PARAM managedBy="system" name="clusterName" required="true" type="string">DM</PARAM>
177 
                    <PARAM managedBy="user" name="tableName" required="true" type="string">dev_openaire</PARAM>
178 
                    <PARAM managedBy="user" name="sql" required="true" type="string">SELECT source_type, source_id, target_type, target_id, semantics FROM claim WHERE approved=TRUE AND source_type !='context'</PARAM>
179 
                    <PARAM managedBy="user" name="countQuery" required="true" type="string">SELECT count(*) FROM claim WHERE approved=TRUE AND source_type !='context'</PARAM>
180 
                </PARAMETERS>
181 
                <ARCS>
182 
                    <ARC to="doneClaimRels"/>
183 
                </ARCS>
184 
            </NODE>
174 185 
            <NODE name="storeDatasourceOrganization" type="StoreHBase">
175 186 
                <DESCRIPTION>Store relation Datasource_Organization</DESCRIPTION>
176 187 
                <PARAMETERS>
177 188 
                    <PARAM managedBy="system" name="inputEprParam" required="true" type="string">dsOrgEpr</PARAM>
178 189 
                    <PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM>
179 190 
                    <PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
180 
                    <PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM>
191 
                    <PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">760183e7-a5d4-4827-9218-3d49ff9fcbd1_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
181 192 
                </PARAMETERS>
182 193 
                <ARCS>
183 194 
                    <ARC to="doneDsOrgs"/>
......
196 207 
                    <PARAM managedBy="system" name="inputEprParam" required="true" type="string">dsEpr</PARAM>
197 208 
                    <PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM>
198 209 
                    <PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
199 
                    <PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM>
210 
                    <PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">30383440-fec9-4889-8ce8-147410d6e63c_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
200 211 
                </PARAMETERS>
201 212 
                <ARCS>
202 213 
                    <ARC to="doneDatasources"/>
......
225 236 
                    <PARAM managedBy="system" name="inputEprParam" required="true" type="string">orgEpr</PARAM>
226 237 
                    <PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM>
227 238 
                    <PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
228 
                    <PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM>
239 
                    <PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">4ef23346-95e3-4357-a4bf-e26d7c9c1b82_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
229 240 
                </PARAMETERS>
230 241 
                <ARCS>
231 242 
                    <ARC to="doneOrganizations"/>
......
238 249 
                    <ARC to="checkpoint"/>
239 250 
                </ARCS>
240 251 
            </NODE>
241 
            <NODE name="SELECT_CLAIM_REL" type="Selection">
242 
                <DESCRIPTION>Do we import the claimed relationships?</DESCRIPTION>
243 
                <PARAMETERS>
244 
                    <PARAM function="validValues(['YES', 'NO'])" managedBy="user" name="selection" required="true" type="string">NO</PARAM>
245 
                </PARAMETERS>
246 
                <ARCS>
247 
                    <ARC name="YES" to="queryClaimRels"/>
248 
                    <ARC name="NO" to="doneClaimRels"/>
249 
                </ARCS>
250 
            </NODE>
251 252 
            <NODE name="storePersons" type="StoreHBase">
252 253 
                <DESCRIPTION>Store Persons to HBase</DESCRIPTION>
253 254 
                <PARAMETERS>
254 255 
                    <PARAM managedBy="system" name="inputEprParam" required="true" type="string">personsEpr</PARAM>
255 256 
                    <PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM>
256 257 
                    <PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
257 
                    <PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM>
258 
                    <PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">dec9fb5d-c845-4ea3-a8a4-1b8c6b2f2682_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
258 259 
                </PARAMETERS>
259 260 
                <ARCS>
260 261 
                    <ARC to="donePersons"/>
......
283 284 
                    <PARAM managedBy="system" name="inputEprParam" required="true" type="string">projOrgEpr</PARAM>
284 285 
                    <PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM>
285 286 
                    <PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
286 
                    <PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM>
287 
                    <PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">621df4c6-ff71-4c1c-82bf-67a64c850be6_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
287 288 
                </PARAMETERS>
288 289 
                <ARCS>
289 290 
                    <ARC to="donePrjOrgs"/>
......
302 303 
                    <PARAM managedBy="system" name="inputEprParam" required="true" type="string">projPerEpr</PARAM>
303 304 
                    <PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM>
304 305 
                    <PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
305 
                    <PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM>
306 
                    <PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">9099673b-b46d-4aae-9f56-321607cbacba_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
306 307 
                </PARAMETERS>
307 308 
                <ARCS>
308 309 
                    <ARC to="donePrjPer"/>
......
321 322 
                    <PARAM managedBy="system" name="inputEprParam" required="true" type="string">projEpr</PARAM>
322 323 
                    <PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM>
323 324 
                    <PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
324 
                    <PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM>
325 
                    <PARAM function="obtainValues('dbmf2hbaseMappings', {})" managedBy="user" name="mapping" required="true" type="string">5fccd61e-e6e5-4912-b849-54bc84b69a83_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</PARAM>
325 326 
                </PARAMETERS>
326 327 
                <ARCS>
327 328 
                    <ARC to="doneProjects"/>
......
334 335 
                    <ARC to="finish"/>
335 336 
                </ARCS>
336 337 
            </NODE>
337 
            <NODE name="storeClaimRels" type="StoreHBase">
338 
                <DESCRIPTION>Store relation Claim Project</DESCRIPTION>
339 
                <PARAMETERS>
340 
                    <PARAM managedBy="system" name="inputEprParam" required="true" type="string">claimRelsEpr</PARAM>
341 
                    <PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM>
342 
                    <PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
343 
                    <PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM>
344 
                </PARAMETERS>
345 
                <ARCS>
346 
                    <ARC to="doneClaimRels"/>
347 
                </ARCS>
348 
            </NODE>
349 338 
            <NODE name="doneClaimRels">
350 339 
                <DESCRIPTION/>
351 340 
                <PARAMETERS/>
......
379 368 
            <LAST_EXECUTION_OUTPUT name="mainlog:storeOrganizations:count">33455</LAST_EXECUTION_OUTPUT>
380 369 
        </STATUS>
381 370 
    </BODY>
382 
</RESOURCE_PROFILE>
371 
</RESOURCE_PROFILE>

Also available in: Unified diff