1
|
package eu.dnetlib.enabling.datasources;
|
2
|
|
3
|
import java.io.StringReader;
|
4
|
import java.util.Collections;
|
5
|
import java.util.Date;
|
6
|
import java.util.HashMap;
|
7
|
import java.util.List;
|
8
|
import java.util.Map;
|
9
|
import java.util.Map.Entry;
|
10
|
|
11
|
import org.apache.commons.lang.StringEscapeUtils;
|
12
|
import org.apache.commons.lang.StringUtils;
|
13
|
import org.apache.commons.lang.math.NumberUtils;
|
14
|
import org.apache.commons.logging.Log;
|
15
|
import org.apache.commons.logging.LogFactory;
|
16
|
import org.dom4j.Document;
|
17
|
import org.dom4j.Node;
|
18
|
import org.dom4j.io.SAXReader;
|
19
|
import org.springframework.beans.factory.annotation.Required;
|
20
|
|
21
|
import com.google.common.base.Function;
|
22
|
import com.google.common.collect.Iterables;
|
23
|
import com.google.common.collect.Lists;
|
24
|
import com.google.common.collect.Maps;
|
25
|
|
26
|
import eu.dnetlib.enabling.datasources.common.Api;
|
27
|
import eu.dnetlib.enabling.datasources.common.BrowseTerm;
|
28
|
import eu.dnetlib.enabling.datasources.common.Datasource;
|
29
|
import eu.dnetlib.enabling.datasources.common.DatasourceManager;
|
30
|
import eu.dnetlib.enabling.datasources.common.DatasourceManagerException;
|
31
|
|
32
|
public class LocalOpenaireDatasourceManager implements DatasourceManager<Datasource<?, ?>, Api> {
|
33
|
|
34
|
private DatasourceManagerClients datasourceManagerClients;
|
35
|
|
36
|
private List<DbBrowsableField> browsableFields;
|
37
|
|
38
|
private static final Log log = LogFactory.getLog(LocalOpenaireDatasourceManager.class);
|
39
|
|
40
|
@Override
|
41
|
public void saveDs(final Datasource<?, ?> ds) throws DatasourceManagerException {
|
42
|
if (StringUtils.isBlank(ds.getAggregator())) {
|
43
|
ds.setAggregator("OPENAIRE");
|
44
|
}
|
45
|
|
46
|
final Map<String, Object> params = DatasourceFunctions.asMapOfSqlValues(ds);
|
47
|
|
48
|
if ((ds.getOrganizations() != null) && !ds.getOrganizations().isEmpty()) {
|
49
|
params.put("hasOrganization", 1);
|
50
|
}
|
51
|
|
52
|
datasourceManagerClients.updateSQL(ds.getId(), "addDatasource.sql.st", params, false, true);
|
53
|
}
|
54
|
|
55
|
@Override
|
56
|
public void deleteDs(final String dsId) throws DatasourceManagerException {
|
57
|
final Map<String, Object> params = Maps.newHashMap();
|
58
|
|
59
|
params.put("dsId", DatasourceFunctions.asSqlValue(dsId));
|
60
|
|
61
|
datasourceManagerClients.updateSQL(dsId, "deleteDatasource.sql.st", params, true, true);
|
62
|
}
|
63
|
|
64
|
@Override
|
65
|
public Datasource<?, ?> getDs(final String dsId) throws DatasourceManagerException {
|
66
|
final List<Datasource<?, ?>> list = datasourceManagerClients.getDatasourcesByCondition("ds.id = " + DatasourceFunctions.asSqlValue(dsId));
|
67
|
if (list.size() != 1) { throw new DatasourceManagerException("Datasource not found, id=" + dsId); }
|
68
|
|
69
|
return list.get(0);
|
70
|
}
|
71
|
|
72
|
@Override
|
73
|
public void updateActivationStatus(final String dsId, final String ifaceId, final boolean active) throws DatasourceManagerException {
|
74
|
final Map<String, Object> params = Maps.newHashMap();
|
75
|
|
76
|
params.put("active", DatasourceFunctions.asSqlValue(active));
|
77
|
params.put("ifaceId", DatasourceFunctions.asSqlValue(ifaceId));
|
78
|
params.put("dsId", DatasourceFunctions.asSqlValue(dsId));
|
79
|
|
80
|
datasourceManagerClients.updateSQL(dsId, "updateActivationStatus.sql.st", params, false, true);
|
81
|
}
|
82
|
|
83
|
@Override
|
84
|
public void updateCompliance(final String dsId, final String ifaceId, final String level) throws DatasourceManagerException {
|
85
|
final Map<String, Object> params = Maps.newHashMap();
|
86
|
|
87
|
params.put("level", DatasourceFunctions.asSqlValue(level));
|
88
|
params.put("ifaceId", DatasourceFunctions.asSqlValue(ifaceId));
|
89
|
params.put("dsId", DatasourceFunctions.asSqlValue(dsId));
|
90
|
|
91
|
datasourceManagerClients.updateSQL(dsId, "updateLevelOfCompliance.sql.st", params, false, true);
|
92
|
}
|
93
|
|
94
|
@Override
|
95
|
public void updateBaseUrl(final String dsId, final String ifaceId, final String baseUrl) throws DatasourceManagerException {
|
96
|
updateAccessParam(dsId, ifaceId, DatasourceParams.baseUrl.toString(), baseUrl);
|
97
|
}
|
98
|
|
99
|
@Override
|
100
|
public void addApi(final Api iface) throws DatasourceManagerException {
|
101
|
final Map<String, Object> params = Maps.newHashMap();
|
102
|
|
103
|
params.put("datasource", DatasourceFunctions.asSqlValue(iface.getDatasource()));
|
104
|
params.put("id", DatasourceFunctions.asSqlValue(iface.getId()));
|
105
|
params.put("typology", DatasourceFunctions.asSqlValue(iface.getTypology()));
|
106
|
params.put("protocol", DatasourceFunctions.asSqlValue(iface.getAccessProtocol()));
|
107
|
params.put("baseUrl", DatasourceFunctions.asSqlValue(iface.getBaseUrl()));
|
108
|
params.put("description", DatasourceFunctions.asSqlValue(iface.getContentDescription()));
|
109
|
params.put("compliance", DatasourceFunctions.asSqlValue(iface.getCompliance()));
|
110
|
|
111
|
final Map<String, String> accessParams = new HashMap<String, String>();
|
112
|
if (iface.getAccessParams() != null) {
|
113
|
for (final Entry<String, String> e : iface.getAccessParams().entrySet()) {
|
114
|
accessParams.put(DatasourceFunctions.asSqlValue(e.getKey()), DatasourceFunctions.asSqlValue(e.getValue()));
|
115
|
}
|
116
|
}
|
117
|
params.put("accessParams", accessParams);
|
118
|
|
119
|
final Map<String, String> extraFields = new HashMap<String, String>();
|
120
|
if (iface.getExtraFields() != null) {
|
121
|
for (final Entry<String, String> e : iface.getExtraFields().entrySet()) {
|
122
|
extraFields.put(DatasourceFunctions.asSqlValue(e.getKey()), DatasourceFunctions.asSqlValue(e.getValue()));
|
123
|
}
|
124
|
}
|
125
|
params.put("extraFields", extraFields);
|
126
|
|
127
|
datasourceManagerClients.updateSQL(dsId, "insertInterface.sql.st", params, false, true);
|
128
|
}
|
129
|
|
130
|
@Override
|
131
|
public void deleteApi(String dsId, final String ifcId) throws DatasourceManagerException {
|
132
|
final Map<String, Object> params = Maps.newHashMap();
|
133
|
|
134
|
params.put("datasource", DatasourceFunctions.asSqlValue(dsId));
|
135
|
params.put("id", DatasourceFunctions.asSqlValue(ifcId));
|
136
|
|
137
|
datasourceManagerClients.updateSQL(dsId, "deleteInterface.sql.st", params, false, true);
|
138
|
}
|
139
|
|
140
|
@Override
|
141
|
public boolean updateAccessParam(final String dsId, final String ifaceId, final String field, final String value) throws DatasourceManagerException {
|
142
|
|
143
|
final String openaireDsId = datasourceManagerClients.findDatasourceId(dsId);
|
144
|
|
145
|
final Map<String, Object> params = Maps.newHashMap();
|
146
|
|
147
|
params.put("dsId", DatasourceFunctions.asSqlValue(openaireDsId));
|
148
|
params.put("ifaceId", DatasourceFunctions.asSqlValue(ifaceId));
|
149
|
params.put("field", DatasourceFunctions.asSqlValue(field));
|
150
|
params.put("accessParam", true);
|
151
|
|
152
|
if (value != null) {
|
153
|
params.put("value", DatasourceFunctions.asSqlValue(value));
|
154
|
}
|
155
|
|
156
|
if (datasourceManagerClients.isDefinedParam(ifaceId, field)) {
|
157
|
params.put("update", 1);
|
158
|
}
|
159
|
|
160
|
return datasourceManagerClients.updateSQL(openaireDsId, "updateApiCollectionsRow.sql.st", params, false, true);
|
161
|
}
|
162
|
|
163
|
@Override
|
164
|
public boolean deleteAccessParam(final String dsId, final String ifaceId, final String field) throws DatasourceManagerException {
|
165
|
final Map<String, Object> params = Maps.newHashMap();
|
166
|
|
167
|
params.put("dsId", DatasourceFunctions.asSqlValue(dsId));
|
168
|
params.put("ifaceId", DatasourceFunctions.asSqlValue(ifaceId));
|
169
|
params.put("field", DatasourceFunctions.asSqlValue(field));
|
170
|
|
171
|
return datasourceManagerClients.updateSQL(dsId, "deleteApiCollectionsRow.sql.st", params, false, true);
|
172
|
}
|
173
|
|
174
|
@Override
|
175
|
public boolean updateSQL(final String dsId, final String sql, final boolean delete) throws DatasourceManagerException {
|
176
|
return datasourceManagerClients.updateSQL(dsId, sql, delete, true);
|
177
|
}
|
178
|
|
179
|
@Override
|
180
|
public Date findNextScheduledExecution(final String dsId, final String ifaceId)
|
181
|
throws DatasourceManagerException {
|
182
|
return datasourceManagerClients.findNextScheduledExecution(dsId, ifaceId);
|
183
|
}
|
184
|
|
185
|
@Override
|
186
|
public boolean bulkUpdateApiAccessParams(final String repoId, final String ifaceId, final Map<String, String> params)
|
187
|
throws DatasourceManagerException {
|
188
|
return performUpdate(repoId, ifaceId, params, true);
|
189
|
}
|
190
|
|
191
|
private boolean performUpdate(final String repoId, final String ifaceId, final Map<String, String> params, final boolean accessParam)
|
192
|
throws DatasourceManagerException {
|
193
|
final String dsId = datasourceManagerClients.findDatasourceId(repoId);
|
194
|
|
195
|
if (!accessParam) {
|
196
|
deleteOldExtraFields(dsId, ifaceId);
|
197
|
}
|
198
|
|
199
|
for (final Map.Entry<String, String> e : params.entrySet()) {
|
200
|
performSingleUpdate(dsId, ifaceId, e.getKey(), e.getValue(), accessParam);
|
201
|
}
|
202
|
datasourceManagerClients.regenerateProfile(dsId);
|
203
|
return true;
|
204
|
}
|
205
|
|
206
|
private boolean deleteOldExtraFields(final String dsId, final String ifaceId) throws DatasourceManagerException {
|
207
|
final Map<String, Object> params = Maps.newHashMap();
|
208
|
|
209
|
params.put("dsId", DatasourceFunctions.asSqlValue(dsId));
|
210
|
params.put("ifaceId", DatasourceFunctions.asSqlValue(ifaceId));
|
211
|
|
212
|
return datasourceManagerClients.updateSQL(dsId, "deleteOldExtraFields.sql.st", params, false, false);
|
213
|
}
|
214
|
|
215
|
private boolean performSingleUpdate(final String dsId, final String ifaceId, final String field, final String value, final boolean accessParam)
|
216
|
throws DatasourceManagerException {
|
217
|
final Map<String, Object> params = Maps.newHashMap();
|
218
|
|
219
|
params.put("dsId", DatasourceFunctions.asSqlValue(dsId));
|
220
|
params.put("ifaceId", DatasourceFunctions.asSqlValue(ifaceId));
|
221
|
params.put("field", DatasourceFunctions.asSqlValue(field));
|
222
|
params.put("accessParam", accessParam);
|
223
|
|
224
|
if (value != null) {
|
225
|
params.put("value", DatasourceFunctions.asSqlValue(value));
|
226
|
}
|
227
|
|
228
|
if (accessParam && datasourceManagerClients.isDefinedParam(ifaceId, field)) {
|
229
|
params.put("update", 1);
|
230
|
params.put("preserveOriginal", 1);
|
231
|
}
|
232
|
|
233
|
return datasourceManagerClients.updateSQL(dsId, "updateApiCollectionsRow.sql.st", params, false, false);
|
234
|
}
|
235
|
|
236
|
public DatasourceManagerClients getDatasourceManagerClients() {
|
237
|
return datasourceManagerClients;
|
238
|
}
|
239
|
|
240
|
@Required
|
241
|
public void setDatasourceManagerClients(final DatasourceManagerClients datasourceManagerClients) {
|
242
|
this.datasourceManagerClients = datasourceManagerClients;
|
243
|
}
|
244
|
|
245
|
@Override
|
246
|
public boolean overrideCompliance(final String repoId, final String ifaceId, final String compliance) throws DatasourceManagerException {
|
247
|
|
248
|
final String dsId = datasourceManagerClients.findDatasourceId(repoId);
|
249
|
|
250
|
final Map<String, Object> params = Maps.newHashMap();
|
251
|
params.put("dsId", DatasourceFunctions.asSqlValue(dsId));
|
252
|
params.put("ifaceId", DatasourceFunctions.asSqlValue(ifaceId));
|
253
|
params.put("field", DatasourceFunctions.asSqlValue(DatasourceConstants.OVERRIDING_COMPLIANCE_FIELD));
|
254
|
|
255
|
if (StringUtils.isEmpty(compliance)) {
|
256
|
params.put("delete", true);
|
257
|
log.debug("Removing compliance");
|
258
|
} else {
|
259
|
params.put("value", DatasourceFunctions.asSqlValue(compliance));
|
260
|
if (datasourceManagerClients.isDefinedParam(ifaceId, DatasourceConstants.OVERRIDING_COMPLIANCE_FIELD)) {
|
261
|
params.put("update", true);
|
262
|
log.debug("Updating compliance: " + compliance);
|
263
|
} else {
|
264
|
params.put("insert", true);
|
265
|
log.debug("Adding compliance: " + compliance);
|
266
|
}
|
267
|
}
|
268
|
return datasourceManagerClients.updateSQL(dsId, "overrideCompliance.sql.st", params, false, true);
|
269
|
}
|
270
|
|
271
|
@Override
|
272
|
public List<BrowsableField> listBrowsableFields() throws DatasourceManagerException {
|
273
|
return Lists.transform(getBrowsableFields(), new Function<DbBrowsableField, BrowsableField>() {
|
274
|
|
275
|
@Override
|
276
|
public BrowsableField apply(final DbBrowsableField f) {
|
277
|
return new BrowsableField(f.getId(), f.getLabel());
|
278
|
}
|
279
|
});
|
280
|
}
|
281
|
|
282
|
@Override
|
283
|
public List<BrowseTerm> browseField(final String f) throws DatasourceManagerException {
|
284
|
final List<BrowseTerm> list = Lists.newArrayList();
|
285
|
|
286
|
try {
|
287
|
final DbBrowsableField field = findBrowseField(f);
|
288
|
if (field != null) {
|
289
|
final SAXReader reader = new SAXReader();
|
290
|
for (final String s : datasourceManagerClients.searchSQL(field.getSql())) {
|
291
|
final Document doc = reader.read(new StringReader(s));
|
292
|
final String id = doc.valueOf("/ROW/FIELD[@name='id']");
|
293
|
final String name = doc.valueOf("/ROW/FIELD[@name='name']");
|
294
|
final int value = NumberUtils.toInt(doc.valueOf("/ROW/FIELD[@name='count']"), 0);
|
295
|
if (StringUtils.isNotEmpty(id)) {
|
296
|
list.add(new BrowseTerm(id, name, value));
|
297
|
}
|
298
|
}
|
299
|
}
|
300
|
} catch (final Exception e) {
|
301
|
log.error("Error browsing field " + f, e);
|
302
|
}
|
303
|
|
304
|
return list;
|
305
|
}
|
306
|
|
307
|
@Override
|
308
|
public List<SearchInterfacesEntry> searchInterface(final String field, final String value) throws DatasourceManagerException {
|
309
|
try {
|
310
|
final Map<String, Object> params = Maps.newHashMap();
|
311
|
params.put("value", value);
|
312
|
|
313
|
if (!field.equalsIgnoreCase("__search__")) {
|
314
|
params.put("field", field);
|
315
|
}
|
316
|
|
317
|
final DbBrowsableField f = findBrowseField(field);
|
318
|
if ((f == null) || f.isText()) {
|
319
|
params.put("delimeter", "'");
|
320
|
}
|
321
|
|
322
|
final SAXReader reader = new SAXReader();
|
323
|
final Iterable<String> iter = datasourceManagerClients.searchSQL("searchRepoInterfaces.sql.st", params);
|
324
|
return Lists.newArrayList(Iterables.transform(iter, new Function<String, SearchInterfacesEntry>() {
|
325
|
|
326
|
@Override
|
327
|
public SearchInterfacesEntry apply(final String s) {
|
328
|
final SearchInterfacesEntry iface = new SearchInterfacesEntry();
|
329
|
try {
|
330
|
final Document doc = reader.read(new StringReader(s));
|
331
|
final String country = doc.valueOf("/ROW/FIELD[@name='country']");
|
332
|
|
333
|
iface.setRepoId(doc.valueOf("/ROW/FIELD[@name='repoid']"));
|
334
|
iface.setRepoCountry(StringUtils.isEmpty(country) ? "-" : country.toUpperCase());
|
335
|
iface.setRepoName(StringEscapeUtils.unescapeXml(doc.valueOf("/ROW/FIELD[@name='reponame']")));
|
336
|
iface.setRepoPrefix(doc.valueOf("/ROW/FIELD[@name='repoprefix']"));
|
337
|
|
338
|
iface.setId(doc.valueOf("/ROW/FIELD[@name='id']"));
|
339
|
iface.setActive(Boolean.valueOf(doc.valueOf("/ROW/FIELD[@name='active']")));
|
340
|
iface.setProtocol(doc.valueOf("/ROW/FIELD[@name='protocol']"));
|
341
|
iface.setCompliance(doc.valueOf("/ROW/FIELD[@name='compliance']"));
|
342
|
iface.setAggrDate(doc.valueOf("/ROW/FIELD[@name='aggrdate']"));
|
343
|
iface.setAggrTotal(NumberUtils.toInt(doc.valueOf("/ROW/FIELD[@name='aggrtotal']"), 0));
|
344
|
} catch (final Exception e) {
|
345
|
log.error(e);
|
346
|
}
|
347
|
return iface;
|
348
|
}
|
349
|
}));
|
350
|
} catch (final Exception e) {
|
351
|
log.error("Error searching field " + field + " - value: " + value, e);
|
352
|
}
|
353
|
return Lists.newArrayList();
|
354
|
}
|
355
|
|
356
|
private DbBrowsableField findBrowseField(final String id) {
|
357
|
for (final DbBrowsableField f : getBrowsableFields()) {
|
358
|
if (f.getId().equals(id)) { return f; }
|
359
|
}
|
360
|
return null;
|
361
|
}
|
362
|
|
363
|
@Override
|
364
|
public List<RepositoryMapEntry> getRepositoryMap() throws DatasourceManagerException {
|
365
|
final SAXReader reader = new SAXReader();
|
366
|
|
367
|
try {
|
368
|
final Iterable<String> iter = datasourceManagerClients.searchSQL("findReposMap.sql.st", null);
|
369
|
return Lists.newArrayList(Iterables.transform(iter, s -> {
|
370
|
final RepositoryMapEntry r = new RepositoryMapEntry();
|
371
|
try {
|
372
|
final Document doc = reader.read(new StringReader(s));
|
373
|
r.setId(doc.valueOf("/ROW/FIELD[@name='id']"));
|
374
|
r.setName(StringEscapeUtils.unescapeXml(doc.valueOf("/ROW/FIELD[@name='name']")));
|
375
|
r.setLat(NumberUtils.toFloat(doc.valueOf("/ROW/FIELD[@name='lat']"), 0));
|
376
|
r.setLng(NumberUtils.toFloat(doc.valueOf("/ROW/FIELD[@name='lng']"), 0));
|
377
|
} catch (final Exception e) {
|
378
|
log.error(e);
|
379
|
}
|
380
|
return r;
|
381
|
}));
|
382
|
} catch (final Exception e) {
|
383
|
log.error("Error obtaing repo map entries", e);
|
384
|
}
|
385
|
return Lists.newArrayList();
|
386
|
}
|
387
|
|
388
|
@Override
|
389
|
public List<SimpleDatasourceDesc> simpleListDatasourcesByType(final String type) throws DatasourceManagerException {
|
390
|
final Map<String, Object> params = Maps.newHashMap();
|
391
|
params.put("type", type);
|
392
|
final List<SimpleDatasourceDesc> list = Lists.newArrayList();
|
393
|
try {
|
394
|
final SAXReader reader = new SAXReader();
|
395
|
for (final String s : datasourceManagerClients.searchSQL("simpleFindRepos.sql.st", params)) {
|
396
|
final Document doc = reader.read(new StringReader(s));
|
397
|
final SimpleDatasourceDesc r = new SimpleDatasourceDesc();
|
398
|
r.setId(doc.valueOf("/ROW/FIELD[@name='id']"));
|
399
|
r.setOrigId(doc.valueOf("/ROW/FIELD[@name='id']"));
|
400
|
r.setName(StringEscapeUtils.unescapeXml(doc.valueOf("/ROW/FIELD[@name='name']")));
|
401
|
for (final Object o : doc.selectNodes("/ROW/FIELD[@name='apis']/ITEM")) {
|
402
|
r.getApis().add(((Node) o).getText());
|
403
|
}
|
404
|
r.setValid(true); // TODO
|
405
|
r.setTypology(type);
|
406
|
list.add(r);
|
407
|
}
|
408
|
Collections.sort(list);
|
409
|
} catch (final Exception e) {
|
410
|
log.error("Error listing repos", e);
|
411
|
}
|
412
|
return list;
|
413
|
}
|
414
|
|
415
|
public List<DbBrowsableField> getBrowsableFields() {
|
416
|
return browsableFields;
|
417
|
}
|
418
|
|
419
|
@Required
|
420
|
public void setBrowsableFields(final List<DbBrowsableField> browsableFields) {
|
421
|
this.browsableFields = browsableFields;
|
422
|
}
|
423
|
}
|