Project

General

Profile

« Previous | Next » 

Revision 56358

Added by Miriam Baglioni almost 5 years ago

added new profile for propagation of community result through semantic relation

View differences:

modules/dnet-openaireplus-workflows/branches/dnet-hadoop/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/propagation/propagationCountryToDatasource.xml
1 
<RESOURCE_PROFILE>
2 
    <HEADER>
3 
        <RESOURCE_IDENTIFIER value="9986e1e0-ce6b-41aa-a49e-46a651c10d6b_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl"/>
4 
        <RESOURCE_TYPE value="WorkflowDSResourceType"/>
5 
        <RESOURCE_KIND value="WorkflowDSResources"/>
6 
        <RESOURCE_URI value=""/>
7 
        <DATE_OF_CREATION value="2018-12-03T10:47:41+00:00"/>
8 
    </HEADER>
9 
    <BODY>
10 
        <WORKFLOW_NAME>Propagate Country from Institutional Repos</WORKFLOW_NAME>
11 
        <WORKFLOW_TYPE>Data Provision</WORKFLOW_TYPE>
12 
        <WORKFLOW_PRIORITY>30</WORKFLOW_PRIORITY>
13 
        <CONFIGURATION start="manual">
14 
            <NODE isStart="true" name="setOutputFile" type="SetEnvParameter">
15 
                <DESCRIPTION>Set the CSV file path on HDFS</DESCRIPTION>
16 
                <PARAMETERS>
17 
                    <PARAM managedBy="system" name="parameterName" required="true" type="string">csvPath</PARAM>
18 
                    <PARAM managedBy="user" name="parameterValue" required="true" type="string">/tmp/db_openaireplus_services_beta.countryPropagation</PARAM>
19 
                </PARAMETERS>
20 
                <ARCS>
21 
                    <ARC to="cleanupCsv"/>
22 
                </ARCS>
23 
            </NODE>
24 
            <NODE isStart="true" name="hadoopConfig" type="SetClusterAndTable">
25 
                <DESCRIPTION>Set table name</DESCRIPTION>
26 
                <PARAMETERS>
27 
                    <PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
28 
                    <PARAM managedBy="system" name="tableParam" required="true" type="string">tableName</PARAM>
29 
                    <PARAM managedBy="user" name="table" required="true" type="string">db_openaireplus_services_beta</PARAM>
30 
                </PARAMETERS>
31 
                <ARCS>
32 
                    <ARC to="cleanupCsv"/>
33 
                </ARCS>
34 
            </NODE>
35 
            <NODE isJoin="true" name="cleanupCsv" type="DeleteHdfsPathJob">
36 
                <DESCRIPTION>CSV files cleanup</DESCRIPTION>
37 
                <PARAMETERS>
38 
                    <PARAM managedBy="system" name="envParams" required="true" type="string">
39 
                        {
40 
                        'path' : 'csvPath',
41 
                        'cluster' : 'cluster'
42 
                        }
43 
                    </PARAM>
44 
                </PARAMETERS>
45 
                <ARCS>
46 
                    <ARC to="propagateCountry"/>
47 
                </ARCS>
48 
            </NODE>
49 
            <NODE isJoin="false" name="propagateCountry" type="SubmitHadoopJob">
50 
                <DESCRIPTION>Propagation of country from institutional repos</DESCRIPTION>
51 
                <PARAMETERS>
52 
                    <PARAM function="validValues(['propagationCountryFromInstitutionalRepositoriesSaveToFile', 'propagationCountryFromInstitutionalRepositories'])" managedBy="user" name="hadoopJob" required="true" type="string">propagationCountryFromInstitutionalRepositories</PARAM>
53 
                    <PARAM managedBy="system" name="envParams" required="true" type="string">
54 
                        {
55 
                        'cluster' : 'cluster',
56 
                        'hbase.mapred.inputtable' : 'tableName',
57 
                        'hbase.mapred.outputtable' : 'tableName',
58 
                        'hbase.mapreduce.inputtable' : 'tableName',
59 
                        'hbase.mapreduce.outputtable' : 'tableName',
60 
                        'mapred.output.dir' : 'csvPath'
61 
                        }
62 
                    </PARAM>
63 
                </PARAMETERS>
64 
                <ARCS>
65 
                    <ARC to="success"/>
66 
                </ARCS>
67 
            </NODE>
68 
        </CONFIGURATION>
69 
        <STATUS>
70 
            <LAST_EXECUTION_ID>wf_20190613_071747_187</LAST_EXECUTION_ID>
71 
            <LAST_EXECUTION_DATE>2019-06-13T07:46:27+00:00</LAST_EXECUTION_DATE>
72 
            <LAST_EXECUTION_STATUS>SUCCESS</LAST_EXECUTION_STATUS>
73 
            <LAST_EXECUTION_ERROR/>
74 
        </STATUS>
75 
    </BODY>
76 
</RESOURCE_PROFILE>
modules/dnet-openaireplus-workflows/branches/dnet-hadoop/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/propagation/propagationProjectResult.xml
1 
<RESOURCE_PROFILE>
2 
    <HEADER>
3 
        <RESOURCE_IDENTIFIER value="6f19d058-326a-417d-b9ee-44eed3c32a35_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl"/>
4 
        <RESOURCE_TYPE value="WorkflowDSResourceType"/>
5 
        <RESOURCE_KIND value="WorkflowDSResources"/>
6 
        <RESOURCE_URI value=""/>
7 
        <DATE_OF_CREATION value="2019-02-20T15:46:09+00:00"/>
8 
    </HEADER>
9 
    <BODY>
10 
        <WORKFLOW_NAME>Propagation of Project Results</WORKFLOW_NAME>
11 
        <WORKFLOW_TYPE>Data Provision</WORKFLOW_TYPE>
12 
        <WORKFLOW_PRIORITY>30</WORKFLOW_PRIORITY>
13 
        <CONFIGURATION start="auto">
14 
            <NODE isStart="true" name="setOutputFile" type="SetEnvParameter">
15 
                <DESCRIPTION>Set the CSV file path on HDFS</DESCRIPTION>
16 
                <PARAMETERS>
17 
                    <PARAM managedBy="system" name="parameterName" required="true" type="string">csvPath</PARAM>
18 
                    <PARAM managedBy="user" name="parameterValue" required="true" type="string">/tmp/db_openaireplus_services_beta.projectPropagation</PARAM>
19 
                </PARAMETERS>
20 
                <ARCS>
21 
                    <ARC to="cleanupCsv"/>
22 
                </ARCS>
23 
            </NODE>
24 
            <NODE isStart="true" name="hadoopConfig" type="SetClusterAndTable">
25 
                <DESCRIPTION>Set table name</DESCRIPTION>
26 
                <PARAMETERS>
27 
                    <PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
28 
                    <PARAM managedBy="system" name="tableParam" required="true" type="string">tableName</PARAM>
29 
                    <PARAM managedBy="user" name="table" required="true" type="string">db_openaireplus_services_beta</PARAM>
30 
                </PARAMETERS>
31 
                <ARCS>
32 
                    <ARC to="cleanupCsv"/>
33 
                </ARCS>
34 
            </NODE>
35 
            <NODE isJoin="true" name="cleanupCsv" type="DeleteHdfsPathJob">
36 
                <DESCRIPTION>CSV files cleanup</DESCRIPTION>
37 
                <PARAMETERS>
38 
                    <PARAM managedBy="system" name="envParams" required="true" type="string">
39 
                        {
40 
                        'path' : 'csvPath',
41 
                        'cluster' : 'cluster'
42 
                        }
43 
                    </PARAM>
44 
                </PARAMETERS>
45 
                <ARCS>
46 
                    <ARC to="propagateProject"/>
47 
                </ARCS>
48 
            </NODE>
49 
            <NODE name="propagateProject" type="SubmitHadoopJob">
50 
                <DESCRIPTION>propagation of project link between research result linked by semantic relations</DESCRIPTION>
51 
                <PARAMETERS>
52 
                    <PARAM function="validValues(['propagationProjectToResult','propagationProjectToResultSaveToFile'])" managedBy="user" name="hadoopJob" required="true" type="string">propagationProjectToResult</PARAM><!-- <PARAM managedBy="system" name="hadoopJob" required="true" type="string">projectToResultJob</PARAM>-->
53 
                    <PARAM managedBy="system" name="envParams" required="true" type="string">
54 
                        {
55 
                        'cluster' : 'cluster',
56 
                        'hbase.mapred.inputtable' : 'tableName',
57 
                        'hbase.mapred.outputtable' : 'tableName',
58 
                        'hbase.mapreduce.inputtable' : 'tableName',
59 
                        'hbase.mapreduce.outputtable' : 'tableName',
60 
                        'mapred.output.dir' : 'csvPath'
61 
                        }
62 
                    </PARAM>
63 
                </PARAMETERS>
64 
                <ARCS>
65 
                    <ARC to="success"/>
66 
                </ARCS>
67 
            </NODE>
68 
        </CONFIGURATION>
69 
        <STATUS>
70 
            <LAST_EXECUTION_ID>wf_20190613_074630_96</LAST_EXECUTION_ID>
71 
            <LAST_EXECUTION_DATE>2019-06-13T08:08:06+00:00</LAST_EXECUTION_DATE>
72 
            <LAST_EXECUTION_STATUS>SUCCESS</LAST_EXECUTION_STATUS>
73 
            <LAST_EXECUTION_ERROR/>
74 
        </STATUS>
75 
    </BODY>
76 
</RESOURCE_PROFILE>
modules/dnet-openaireplus-workflows/branches/dnet-hadoop/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/propagation/propagationResult2community.xml
1 
<RESOURCE_PROFILE>
2 
    <HEADER>
3 
        <RESOURCE_IDENTIFIER value="36ab117d-bafd-46a0-9c7e-38736d1caaaf_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl"/>
4 
        <RESOURCE_TYPE value="WorkflowDSResourceType"/>
5 
        <RESOURCE_KIND value="WorkflowDSResources"/>
6 
        <RESOURCE_URI value=""/>
7 
        <DATE_OF_CREATION value="2019-06-28T12:09:03+02:00"/>
8 
    </HEADER>
9 
    <BODY>
10 
        <WORKFLOW_NAME>Propagation result 2 community</WORKFLOW_NAME>
11 
        <WORKFLOW_TYPE>Data Provision</WORKFLOW_TYPE>
12 
        <WORKFLOW_PRIORITY>30</WORKFLOW_PRIORITY>
13 
        <CONFIGURATION start="manual">
14 
            <NODE isStart="true" name="setOutputFile" type="SetEnvParameter">
15 
                <DESCRIPTION>Set the CSV file path on HDFS</DESCRIPTION>
16 
                <PARAMETERS>
17 
                    <PARAM managedBy="system" name="parameterName" required="true" type="string">csvPath</PARAM>
18 
                    <PARAM managedBy="user" name="parameterValue" required="true" type="string">/tmp/db_openaireplus_services_beta.communityPropagation</PARAM>
19 
                </PARAMETERS>
20 
                <ARCS>
21 
                    <ARC to="cleanupCsv"/>
22 
                </ARCS>
23 
            </NODE>
24 
            <NODE isStart="true" name="hadoopConfig" type="SetClusterAndTable">
25 
                <DESCRIPTION>Set table name</DESCRIPTION>
26 
                <PARAMETERS>
27 
                    <PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
28 
                    <PARAM managedBy="system" name="tableParam" required="true" type="string">tableName</PARAM>
29 
                    <PARAM managedBy="user" name="table" required="true" type="string">dev_openaire</PARAM>
30 
                </PARAMETERS>
31 
                <ARCS>
32 
                    <ARC to="cleanupCsv"/>
33 
                </ARCS>
34 
            </NODE>
35 
            <NODE isJoin="true" name="cleanupCsv" type="DeleteHdfsPathJob">
36 
                <DESCRIPTION>CSV files cleanup</DESCRIPTION>
37 
                <PARAMETERS>
38 
                    <PARAM managedBy="system" name="envParams" required="true" type="string">
39 
                        {
40 
                        'path' : 'csvPath',
41 
                        'cluster' : 'cluster'
42 
                        }
43 
                    </PARAM>
44 
                </PARAMETERS>
45 
                <ARCS>
46 
                    <ARC to="fetchCommunityIdListr"/>
47 
                </ARCS>
48 
            </NODE>
49 
            <NODE isStart="true" name="fetchCommunityIdListr" type="LoadCommunityIdsList">
50 
                <DESCRIPTION>Fetch the list of the community ids</DESCRIPTION>
51 
                <PARAMETERS>
52 
                    <PARAM managedBy="system" name="communityIdParam" required="true">community.ids</PARAM>
53 
                </PARAMETERS>
54 
                <ARCS>
55 
                    <ARC to="propagateCommunity"/>
56 
                </ARCS>
57 
            </NODE>
58 
            <NODE isJoin="true" name="propagate" type="SubmitHadoopJob">
59 
                <DESCRIPTION>Propagatio of result to community through semantic relation</DESCRIPTION>
60 
                <PARAMETERS>
61 
                    <PARAM function="validValues(['propagationResultToCommunityThroughSemanticRelation','propagationResultToCommunityThroughSemanticRelationSaveToFile'])" managedBy="user" name="hadoopJob" required="true" type="string">propagationResultToCommunityThroughSemanticRelationSaveToFile</PARAM>
62 
                    <PARAM managedBy="system" name="envParams" required="true" type="string">
63 
                        {
64 
                        'cluster' : 'cluster',
65 
                        'hbase.mapred.inputtable' : 'tableName',
66 
                        'hbase.mapred.outputtable' : 'tableName',
67 
                        'hbase.mapreduce.inputtable' : 'tableName',
68 
                        'hbase.mapreduce.outputtable' : 'tableName',
69 
                        'community.ids' : 'community.ids'
70 
                        }
71 
                    </PARAM>
72 
                </PARAMETERS>
73 
                <ARCS>
74 
                    <ARC to="success"/>
75 
                </ARCS>
76 
            </NODE>
77 
        </CONFIGURATION>
78 
        <STATUS>
79 
            <LAST_EXECUTION_ID>wf_20180807_172327_510</LAST_EXECUTION_ID>
80 
            <LAST_EXECUTION_DATE>2019-06-28T17:24:53+02:00</LAST_EXECUTION_DATE>
81 
            <LAST_EXECUTION_STATUS>SUCCESS</LAST_EXECUTION_STATUS>
82 
            <LAST_EXECUTION_ERROR/>
83 
        </STATUS>
84 
    </BODY>
85 
</RESOURCE_PROFILE>

Also available in: Unified diff