Revision 57383
Added by Michele Artini almost 5 years ago
modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/db2hbase.xml | ||
---|---|---|
91 | 91 |
<PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM> |
92 | 92 |
</PARAMETERS> |
93 | 93 |
<ARCS> |
94 |
<ARC to="queryOrganizationsFromOpenOrgsDB"/> |
|
95 |
</ARCS> |
|
96 |
</NODE> |
|
97 |
<NODE name="queryOrganizationsFromOpenOrgsDB" type="QueryDb"> |
|
98 |
<DESCRIPTION>query Open Organizations</DESCRIPTION> |
|
99 |
<PARAMETERS> |
|
100 |
<PARAM managedBy="system" name="dbProperty" required="true" type="string">dnet.openorgs.db.name</PARAM> |
|
101 |
<PARAM managedBy="system" name="sql" required="true" type="string">/eu/dnetlib/msro/openaireplus/workflows/hbase/queryOrganizationsFromOpenOrgsDB.sql |
|
102 |
</PARAM> |
|
103 |
<PARAM managedBy="system" name="outputEprParam" required="true" type="string">openOrgsEpr</PARAM> |
|
104 |
</PARAMETERS> |
|
105 |
<ARCS> |
|
106 |
<ARC to="storeOrganizationsFromOpenOrgsDB"/> |
|
107 |
</ARCS> |
|
108 |
</NODE> |
|
109 |
<NODE name="storeOrganizationsFromOpenOrgsDB" type="StoreHBase"> |
|
110 |
<DESCRIPTION>Store Open Organizations to HBase</DESCRIPTION> |
|
111 |
<PARAMETERS> |
|
112 |
<PARAM managedBy="system" name="inputEprParam" required="true" type="string">openOrgsEpr</PARAM> |
|
113 |
<PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM> |
|
114 |
<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM> |
|
115 |
<PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM> |
|
116 |
</PARAMETERS> |
|
117 |
<ARCS> |
|
94 | 118 |
<ARC to="queryProjects"/> |
95 | 119 |
</ARCS> |
96 | 120 |
</NODE> |
modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/msro/openaireplus/applicationContext-msro-openaireplus.properties | ||
---|---|---|
80 | 80 |
dnet.openaire.propagation.xquery = for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')\ |
81 | 81 |
where $x//CONFIGURATION/context[./@type='community' or ./@type='ri']\ |
82 | 82 |
and $x//CONFIGURATION/context/param[./@name='status']/text() != 'hidden'\ |
83 |
return $x//CONFIGURATION/context/@id/string() |
|
83 |
return $x//CONFIGURATION/context/@id/string() |
|
84 |
|
|
85 |
dnet.openorgs.db.name=dev_orgs |
|
86 |
|
modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryOrganizationsFromOpenOrgsDB.sql | ||
---|---|---|
1 |
SELECT |
|
2 |
o.id AS organizationid, |
|
3 |
coalesce((array_agg(a.acronym))[1], o.name) AS legalshortname, |
|
4 |
o.name AS legalname, |
|
5 |
array_agg(DISTINCT n.name) AS "alternativeNames", |
|
6 |
(array_agg(u.url))[1] AS websiteurl, |
|
7 |
o.modification_date AS dateoftransformation, |
|
8 |
false AS inferred, |
|
9 |
false AS deletedbyinference, |
|
10 |
0.95 AS trust, |
|
11 |
'' AS inferenceprovenance, |
|
12 |
'openaire____::openorgs' AS collectedfromid, |
|
13 |
'OpenOrgs Database' AS collectedfromname, |
|
14 |
o.country || '@@@dnet:countries' AS country, |
|
15 |
'sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions' AS provenanceaction, |
|
16 |
array_agg(DISTINCT i.otherid || '###' || i.type) AS pid |
|
17 |
FROM organizations o |
|
18 |
LEFT OUTER JOIN acronyms a ON (a.id = o.id) |
|
19 |
LEFT OUTER JOIN urls u ON (u.id = o.id) |
|
20 |
LEFT OUTER JOIN other_ids i ON (i.id = o.id) |
|
21 |
LEFT OUTER JOIN other_names n ON (n.id = o.id) |
|
22 |
GROUP BY |
|
23 |
o.id, |
|
24 |
o.name, |
|
25 |
o.modification_date, |
|
26 |
o.country |
|
27 |
|
|
28 |
|
Also available in: Unified diff
Import of OpenOrgs Organizations