Project

General

Profile

« Previous | Next » 

Revision 57383

Import of OpenOrgs Organizations

View differences:

modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/db2hbase.xml
91 91
					<PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM>
92 92
				</PARAMETERS>
93 93
				<ARCS>
94
					<ARC to="queryOrganizationsFromOpenOrgsDB"/>
95
				</ARCS>
96
			</NODE>
97
			<NODE name="queryOrganizationsFromOpenOrgsDB" type="QueryDb">
98
				<DESCRIPTION>query Open Organizations</DESCRIPTION>
99
				<PARAMETERS>
100
					<PARAM managedBy="system" name="dbProperty" required="true" type="string">dnet.openorgs.db.name</PARAM>
101
					<PARAM managedBy="system" name="sql" required="true" type="string">/eu/dnetlib/msro/openaireplus/workflows/hbase/queryOrganizationsFromOpenOrgsDB.sql
102
					</PARAM>
103
					<PARAM managedBy="system" name="outputEprParam" required="true" type="string">openOrgsEpr</PARAM>
104
				</PARAMETERS>
105
				<ARCS>
106
					<ARC to="storeOrganizationsFromOpenOrgsDB"/>
107
				</ARCS>
108
			</NODE>
109
			<NODE name="storeOrganizationsFromOpenOrgsDB" type="StoreHBase">
110
				<DESCRIPTION>Store Open Organizations to HBase</DESCRIPTION>
111
				<PARAMETERS>
112
					<PARAM managedBy="system" name="inputEprParam" required="true" type="string">openOrgsEpr</PARAM>
113
					<PARAM managedBy="system" name="hbaseTableProperty" required="true" type="string">hbase.mapred.datatable</PARAM>
114
					<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
115
					<PARAM managedBy="user" name="mapping" required="true" type="string" function="obtainValues('dbmf2hbaseMappings', {})"></PARAM>
116
				</PARAMETERS>
117
				<ARCS>
94 118
					<ARC to="queryProjects"/>
95 119
				</ARCS>
96 120
			</NODE>
modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/msro/openaireplus/applicationContext-msro-openaireplus.properties
80 80
dnet.openaire.propagation.xquery = for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')\
81 81
  where $x//CONFIGURATION/context[./@type='community' or ./@type='ri']\
82 82
  and  $x//CONFIGURATION/context/param[./@name='status']/text() != 'hidden'\
83
  return $x//CONFIGURATION/context/@id/string()
83
  return $x//CONFIGURATION/context/@id/string()
84
  
85
  dnet.openorgs.db.name=dev_orgs
86
  
modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryOrganizationsFromOpenOrgsDB.sql
1
SELECT
2
	o.id                                                                                                                          AS organizationid,
3
	coalesce((array_agg(a.acronym))[1], o.name)                                                                                   AS legalshortname,
4
	o.name                                                                                                                        AS legalname,
5
	array_agg(DISTINCT n.name)                                                                                                    AS "alternativeNames",
6
	(array_agg(u.url))[1]                                                                                                         AS websiteurl,
7
	o.modification_date                                                                                                           AS dateoftransformation,
8
	false                                                                                                                         AS inferred,
9
	false                                                                                                                         AS deletedbyinference,
10
	0.95                                                                                                                          AS trust,
11
	''                                                                                                                            AS inferenceprovenance,
12
	'openaire____::openorgs'                                                                                                      AS collectedfromid,
13
	'OpenOrgs Database'                                                                                                           AS collectedfromname,
14
	o.country || '@@@dnet:countries'                                                                                              AS country,
15
	'sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions' AS provenanceaction,
16
	array_agg(DISTINCT i.otherid || '###' || i.type)                                                                              AS pid
17
FROM organizations o
18
	LEFT OUTER JOIN acronyms a    ON (a.id = o.id)
19
	LEFT OUTER JOIN urls u        ON (u.id = o.id)
20
	LEFT OUTER JOIN other_ids i   ON (i.id = o.id)
21
	LEFT OUTER JOIN other_names n ON (n.id = o.id)
22
GROUP BY
23
	o.id,
24
	o.name,
25
	o.modification_date,
26
	o.country
27

  
28

  

Also available in: Unified diff