1 |
53133
|
claudio.at
|
-- noinspection SqlNoDataSourceInspectionForFile
|
2 |
|
|
|
3 |
47021
|
claudio.at
|
-- cleanup the unnecessary functions: the following SQL produces the deletion statement list
|
4 |
53133
|
claudio.at
|
SELECT 'DROP FUNCTION ' || ns.nspname || '.' || proname || '(' || oidvectortypes(proargtypes) || ') CASCADE;'
|
5 |
48139
|
alessia.ba
|
FROM pg_proc
|
6 |
|
|
INNER JOIN pg_namespace ns ON (pg_proc.pronamespace = ns.oid)
|
7 |
|
|
WHERE ns.nspname = 'public'
|
8 |
|
|
ORDER BY proname;
|
9 |
47021
|
claudio.at
|
|
10 |
48165
|
claudio.at
|
-- cleanup the unnecessary triggers: the following SQL produces the deletion statement list
|
11 |
|
|
SELECT 'DROP TRIGGER ' || trg.tgname || ' ON ' || tbl.relname || ';'
|
12 |
|
|
FROM pg_trigger trg
|
13 |
|
|
JOIN pg_class tbl on trg.tgrelid = tbl.oid
|
14 |
|
|
JOIN pg_namespace ns ON ns.oid = tbl.relnamespace
|
15 |
|
|
WHERE trg.tgname like 'notify_%';
|
16 |
|
|
|
17 |
47021
|
claudio.at
|
-- export the new dnet_dsm schema using
|
18 |
48165
|
claudio.at
|
-- pg_dump --schema-only -h localhost -U dnet dnet_dsm | gzip -c > /tmp/dnet_dsm.sql.gz
|
19 |
47021
|
claudio.at
|
-- rename tables and index names in order to include prefix "dsm_"
|
20 |
|
|
-- import the dsm table schema to the existing openaire database:
|
21 |
48165
|
claudio.at
|
-- psql -h localhost -U dnet dnet_openaireplus -f dnet_dsm.sql
|
22 |
47021
|
claudio.at
|
|
23 |
48518
|
alessia.ba
|
ALTER TABLE dsm_datasources ALTER COLUMN officialname TYPE VARCHAR(512);
|
24 |
|
|
ALTER TABLE dsm_datasources ALTER COLUMN englishname TYPE VARCHAR(512);
|
25 |
|
|
ALTER TABLE dsm_datasources ALTER COLUMN missionstatementurl TYPE VARCHAR(512);
|
26 |
|
|
ALTER TABLE dsm_datasources ALTER COLUMN citationguidelineurl TYPE VARCHAR(512);
|
27 |
|
|
ALTER TABLE dsm_datasources ALTER COLUMN languages TYPE text;
|
28 |
|
|
ALTER TABLE dsm_datasources ALTER COLUMN od_contenttypes TYPE text;
|
29 |
|
|
ALTER TABLE dsm_datasources ALTER COLUMN subjects TYPE text;
|
30 |
|
|
ALTER TABLE dsm_datasources ALTER COLUMN description TYPE text;
|
31 |
|
|
ALTER TABLE dsm_datasources ALTER COLUMN certificates TYPE text;
|
32 |
|
|
ALTER TABLE dsm_datasources ALTER COLUMN pidsystems TYPE text;
|
33 |
|
|
|
34 |
47021
|
claudio.at
|
-- insert into the new dsm_* tables picking the information from the old tables
|
35 |
48518
|
alessia.ba
|
insert into dsm_datasources (id, officialname, englishname, websiteurl, logourl, contactemail, latitude, longitude, timezone, namespaceprefix, languages, od_contenttypes, collectedfrom, dateofvalidation,
|
36 |
|
|
typology, provenanceaction, dateofcollection, platform, activationid, description,releasestartdate, releaseenddate, missionstatementurl, dataprovider, serviceprovider,
|
37 |
|
|
databaseaccesstype, datauploadtype,databaseaccessrestriction, datauploadrestriction, versioning, citationguidelineurl, qualitymanagementkind,pidsystems, certificates,
|
38 |
|
|
aggregator, issn, eissn, lissn, registeredby, subjects)
|
39 |
48165
|
claudio.at
|
select d.id,
|
40 |
|
|
d.officialname,
|
41 |
|
|
d.englishname,
|
42 |
|
|
d.websiteurl,
|
43 |
|
|
d.logourl,
|
44 |
|
|
d.contactemail,
|
45 |
|
|
d.latitude,
|
46 |
|
|
d.longitude,
|
47 |
|
|
d.timezone,
|
48 |
48518
|
alessia.ba
|
d.namespaceprefix as namespaceprefix,
|
49 |
51516
|
claudio.at
|
d.od_languages as languages,
|
50 |
|
|
d.od_contenttypes,
|
51 |
48165
|
claudio.at
|
d.collectedfrom,
|
52 |
|
|
d.dateofvalidation,
|
53 |
|
|
d.datasourceclass as typology,
|
54 |
|
|
d.provenanceactionclass as provenanceaction,
|
55 |
|
|
d.dateofcollection,
|
56 |
|
|
d.typology as platform,
|
57 |
|
|
d.activationid,
|
58 |
|
|
d.description,
|
59 |
|
|
d.releasestartdate,
|
60 |
|
|
d.releaseenddate,
|
61 |
|
|
d.missionstatementurl,
|
62 |
48518
|
alessia.ba
|
d.dataprovider as dataprovider,
|
63 |
|
|
d.serviceprovider as serviceprovider,
|
64 |
48165
|
claudio.at
|
d.databaseaccesstype,
|
65 |
|
|
d.datauploadtype,
|
66 |
|
|
d.databaseaccessrestriction,
|
67 |
|
|
d.datauploadrestriction,
|
68 |
|
|
d.versioning,
|
69 |
|
|
d.citationguidelineurl,
|
70 |
|
|
d.qualitymanagementkind,
|
71 |
|
|
d.pidsystems,
|
72 |
|
|
d.certificates,
|
73 |
|
|
d.aggregator,
|
74 |
|
|
d.issn,
|
75 |
|
|
d.eissn,
|
76 |
|
|
d.lissn,
|
77 |
|
|
d.registeredby,
|
78 |
|
|
ARRAY_AGG(s.name) as subjects
|
79 |
|
|
from datasources d
|
80 |
|
|
left outer join datasource_subject sd on (d.id = sd.datasource)
|
81 |
|
|
left outer join subjects s on (sd.subject = s.id)
|
82 |
|
|
group by
|
83 |
|
|
d.id;
|
84 |
47021
|
claudio.at
|
|
85 |
48518
|
alessia.ba
|
insert into dsm_api (id, protocol, datasource, contentdescription, active, removable, typology, compatibility)
|
86 |
48165
|
claudio.at
|
select a.id,
|
87 |
|
|
a.protocolclass as protocol,
|
88 |
|
|
a.datasource,
|
89 |
|
|
a.contentdescriptionclass as contentdescription,
|
90 |
|
|
a.active,
|
91 |
|
|
a.removable,
|
92 |
|
|
a.typologyclass as typology,
|
93 |
|
|
a.compatibilityclass as compatibility
|
94 |
|
|
from api a;
|
95 |
47021
|
claudio.at
|
|
96 |
48165
|
claudio.at
|
update dsm_api da set metadata_identifier_path = ac.original from apicollections ac where da.id = ac.api and ac.param = 'metadata_identifier_path' ;
|
97 |
|
|
update dsm_api da set last_collection_total = ac.original::integer from apicollections ac where da.id = ac.api and ac.param = 'last_collection_total' ;
|
98 |
|
|
update dsm_api da set last_collection_date = ac.original::date from apicollections ac where da.id = ac.api and ac.param = 'last_collection_date' ;
|
99 |
48651
|
claudio.at
|
update dsm_api da set last_collection_mdid = ac.original from apicollections ac where da.id = ac.api and ac.param = 'last_collection_mdId' ;
|
100 |
48165
|
claudio.at
|
update dsm_api da set last_aggregation_total = ac.original::integer from apicollections ac where da.id = ac.api and ac.param = 'last_aggregation_total' ;
|
101 |
|
|
update dsm_api da set last_aggregation_date = ac.original::date from apicollections ac where da.id = ac.api and ac.param = 'last_aggregation_date' ;
|
102 |
48651
|
claudio.at
|
update dsm_api da set last_aggregation_mdid = ac.original from apicollections ac where da.id = ac.api and ac.param = 'last_aggregation_mdId' ;
|
103 |
48165
|
claudio.at
|
update dsm_api da set last_download_total = ac.original::integer from apicollections ac where da.id = ac.api and ac.param = 'last_download_total' ;
|
104 |
|
|
update dsm_api da set last_download_date = ac.original::date from apicollections ac where da.id = ac.api and ac.param = 'last_download_date' ;
|
105 |
48651
|
claudio.at
|
update dsm_api da set last_download_objid = ac.original from apicollections ac where da.id = ac.api and ac.param = 'last_download_objId' ;
|
106 |
48165
|
claudio.at
|
update dsm_api da set last_validation_job = ac.original from apicollections ac where da.id = ac.api and ac.param = 'last_validation_job' ;
|
107 |
|
|
update dsm_api da set baseurl = CASE WHEN (ac.edited is not null and ac.edited != '') THEN ac.edited ELSE ac.original END from apicollections ac where da.id = ac.api and ac.param = 'baseUrl' ;
|
108 |
47021
|
claudio.at
|
|
109 |
48518
|
alessia.ba
|
insert into dsm_apiparams (param,value, api)
|
110 |
48165
|
claudio.at
|
select
|
111 |
|
|
a.param,
|
112 |
|
|
CASE
|
113 |
|
|
WHEN (a.edited is not null and a.edited != '') THEN a.edited ELSE a.original
|
114 |
|
|
END AS value,
|
115 |
|
|
a.api
|
116 |
|
|
from apicollections a
|
117 |
|
|
where param != 'baseUrl' AND accessparam is true;
|
118 |
47021
|
claudio.at
|
|
119 |
48518
|
alessia.ba
|
|
120 |
|
|
INSERT INTO dsm_organizations (id, legalshortname,legalname,websiteurl, logourl,ec_legalbody, ec_legalperson, ec_nonprofit, ec_researchorganization, ec_highereducation, ec_internationalorganizationeurinterests,
|
121 |
|
|
ec_internationalorganization, ec_enterprise, ec_smevalidated, ec_nutscode, country, collectedfrom, dateofcollection, provenanceaction)
|
122 |
48139
|
alessia.ba
|
SELECT
|
123 |
|
|
o.id,
|
124 |
|
|
o.legalshortname,
|
125 |
|
|
o.legalname,
|
126 |
|
|
o.websiteurl,
|
127 |
|
|
o.logourl,
|
128 |
|
|
o.ec_legalbody,
|
129 |
|
|
o.ec_legalperson,
|
130 |
|
|
o.ec_nonprofit,
|
131 |
|
|
o.ec_researchorganization,
|
132 |
|
|
o.ec_highereducation,
|
133 |
|
|
o.ec_internationalorganizationeurinterests,
|
134 |
|
|
o.ec_internationalorganization,
|
135 |
|
|
o.ec_enterprise,
|
136 |
|
|
o.ec_smevalidated,
|
137 |
|
|
o.ec_nutscode,
|
138 |
|
|
o.countryclass AS country,
|
139 |
|
|
o.collectedfrom,
|
140 |
|
|
o.dateofcollection,
|
141 |
|
|
o.provenanceactionclass AS provenanceaction
|
142 |
|
|
FROM organizations o;
|
143 |
47021
|
claudio.at
|
|
144 |
48518
|
alessia.ba
|
INSERT INTO dsm_datasource_organization (datasource, organization)
|
145 |
48139
|
alessia.ba
|
SELECT
|
146 |
|
|
dorg.datasource,
|
147 |
|
|
dorg.organization
|
148 |
|
|
FROM datasource_organization dorg;
|
149 |
47021
|
claudio.at
|
|
150 |
|
|
|
151 |
48518
|
alessia.ba
|
INSERT INTO dsm_identities (pid, issuertype)
|
152 |
48139
|
alessia.ba
|
SELECT
|
153 |
|
|
i.pid,
|
154 |
|
|
i.issuertypeclass AS issuertype
|
155 |
|
|
FROM identities i;
|
156 |
47021
|
claudio.at
|
|
157 |
48518
|
alessia.ba
|
INSERT INTO dsm_datasourcepids (datasource, pid)
|
158 |
48139
|
alessia.ba
|
SELECT
|
159 |
|
|
dp.datasource,
|
160 |
|
|
dp.pid
|
161 |
|
|
FROM datasourcepids dp;
|
162 |
47021
|
claudio.at
|
|
163 |
48518
|
alessia.ba
|
-- is this table needed? --
|
164 |
|
|
INSERT INTO dsm_organizationpids (organization, pid)
|
165 |
48139
|
alessia.ba
|
SELECT
|
166 |
|
|
op.organization,
|
167 |
|
|
op.pid
|
168 |
|
|
FROM organizationpids op;
|
169 |
47021
|
claudio.at
|
|
170 |
|
|
|
171 |
50724
|
claudio.at
|
-- Migrate the managed status
|
172 |
53133
|
claudio.at
|
update dsm_datasources d set managed = true where d.id in (
|
173 |
50724
|
claudio.at
|
SELECT d.id
|
174 |
|
|
FROM dsm_datasources d
|
175 |
|
|
LEFT OUTER JOIN dsm_api a ON (d.id = a.datasource)
|
176 |
|
|
LEFT OUTER JOIN apicollections ap ON (ap.api = a.id)
|
177 |
|
|
WHERE a.active = TRUE
|
178 |
|
|
OR a.removable = TRUE
|
179 |
|
|
OR ap.edited IS NOT NULL
|
180 |
|
|
GROUP BY d.id );
|
181 |
47021
|
claudio.at
|
|
182 |
|
|
|
183 |
50724
|
claudio.at
|
-- !!!!! data insert complete !!!!!
|
184 |
47021
|
claudio.at
|
|
185 |
|
|
|
186 |
50724
|
claudio.at
|
delete from organizations where id like 'opendoar%' ;
|
187 |
|
|
delete from organizations where id like 're3data%' ;
|
188 |
|
|
delete from organizations where collectedfrom is null ;
|
189 |
47021
|
claudio.at
|
|
190 |
50724
|
claudio.at
|
delete from dsm_organizations where collectedfrom = 'openaire____::corda' ;
|
191 |
|
|
delete from dsm_organizations where collectedfrom = 'openaire____::snsf' ;
|
192 |
|
|
delete from dsm_organizations where collectedfrom = 'openaire____::conicytf' ;
|
193 |
|
|
delete from dsm_organizations where collectedfrom = 'openaire____::irb_hr' ;
|
194 |
|
|
delete from dsm_organizations where collectedfrom = 'openaire____::fwf' ;
|
195 |
|
|
delete from dsm_organizations where collectedfrom = 'openaire____::sfi' ;
|
196 |
|
|
delete from dsm_organizations where collectedfrom = 'openaire____::wellcometrust' ;
|
197 |
|
|
delete from dsm_organizations where collectedfrom = 'openaire____::nsf' ;
|
198 |
|
|
delete from dsm_organizations where collectedfrom = 'openaire____::corda_h2020' ;
|
199 |
47021
|
claudio.at
|
|
200 |
|
|
|
201 |
|
|
|
202 |
|
|
|
203 |
|
|
|
204 |
|
|
|
205 |
|
|
|