1
|
package eu.dnetlib.enabling.datasources;
|
2
|
|
3
|
import java.io.StringWriter;
|
4
|
import java.util.*;
|
5
|
import java.util.stream.Collectors;
|
6
|
|
7
|
import com.google.common.collect.ImmutableMap;
|
8
|
import eu.dnetlib.enabling.datasources.DatasourceManagerClients.AfterSqlUpdate;
|
9
|
import eu.dnetlib.enabling.datasources.common.*;
|
10
|
import org.apache.commons.io.IOUtils;
|
11
|
import org.apache.commons.lang3.StringUtils;
|
12
|
import org.apache.commons.lang3.math.NumberUtils;
|
13
|
import org.apache.commons.logging.Log;
|
14
|
import org.apache.commons.logging.LogFactory;
|
15
|
import org.springframework.beans.factory.annotation.Required;
|
16
|
import org.springframework.core.io.ClassPathResource;
|
17
|
import org.springframework.core.io.Resource;
|
18
|
import org.springframework.transaction.annotation.Transactional;
|
19
|
|
20
|
public class LocalOpenaireDatasourceManagerImpl implements LocalOpenaireDatasourceManager {
|
21
|
|
22
|
private DatasourceManagerClients datasourceManagerClients;
|
23
|
|
24
|
private List<DbBrowsableField> browsableFields;
|
25
|
|
26
|
public static final String QUERY_BASEDIR = "/eu/dnetlib/enabling/datasources/queries/";
|
27
|
|
28
|
private static final Resource searchDsByType = new ClassPathResource(QUERY_BASEDIR + "searchDsByType.sql");
|
29
|
private static final Resource searchApis = new ClassPathResource(QUERY_BASEDIR + "searchApisNormal.sql");
|
30
|
private static final Resource searchApisUsingField = new ClassPathResource(QUERY_BASEDIR + "searchApisUsingField.sql");
|
31
|
private static final Resource addDs = new ClassPathResource(QUERY_BASEDIR + "addDatasource.sql");
|
32
|
private static final Resource addOrg = new ClassPathResource(QUERY_BASEDIR + "addOrganization.sql");
|
33
|
private static final Resource deleteDs = new ClassPathResource(QUERY_BASEDIR + "deleteDatasource.sql");
|
34
|
private static final Resource setActive = new ClassPathResource(QUERY_BASEDIR + "setActive.sql");
|
35
|
private static final Resource setManaged = new ClassPathResource(QUERY_BASEDIR + "setManaged.sql");
|
36
|
private static final Resource setCompliance = new ClassPathResource(QUERY_BASEDIR + "setCompliance.sql");
|
37
|
private static final Resource overrideCompliance = new ClassPathResource(QUERY_BASEDIR + "overrideCompliance.sql");
|
38
|
private static final Resource resetCompliance = new ClassPathResource(QUERY_BASEDIR + "resetCompliance.sql");
|
39
|
private static final Resource setLastCollectionInfo = new ClassPathResource(QUERY_BASEDIR + "setLastCollectionInfo.sql");
|
40
|
private static final Resource setLastAggregationInfo = new ClassPathResource(QUERY_BASEDIR + "setLastAggregationInfo.sql");
|
41
|
private static final Resource setLastDownloadInfo = new ClassPathResource(QUERY_BASEDIR + "setLastDownloadInfo.sql");
|
42
|
private static final Resource resetLastOperationsInfo = new ClassPathResource(QUERY_BASEDIR + "resetLastOperationsInfo.sql");
|
43
|
private static final Resource insertApiParam = new ClassPathResource(QUERY_BASEDIR + "insertApiParam.sql");
|
44
|
private static final Resource insertApi = new ClassPathResource(QUERY_BASEDIR + "insertApi.sql");
|
45
|
|
46
|
private static final Log log = LogFactory.getLog(LocalOpenaireDatasourceManagerImpl.class);
|
47
|
|
48
|
private static final String REPO_PROFILEID_SUFFIX = "_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=";
|
49
|
|
50
|
@Override
|
51
|
@Transactional(readOnly = true)
|
52
|
public Set<String> listManagedDatasourceIds() throws DsmRuntimeException {
|
53
|
try {
|
54
|
return datasourceManagerClients.searchSQL("SELECT id FROM dsm_datasources WHERE managed = true", new HashMap<>())
|
55
|
.stream()
|
56
|
.map(m -> (String) m.get("id"))
|
57
|
.collect(Collectors.toCollection(HashSet::new));
|
58
|
} catch (final DsmException e) {
|
59
|
throw new DsmRuntimeException(e);
|
60
|
}
|
61
|
}
|
62
|
|
63
|
@Override
|
64
|
@Transactional(readOnly = true)
|
65
|
public List<SimpleDatasource> searchDatasourcesByType(final String type) throws DsmException {
|
66
|
|
67
|
return datasourceManagerClients.searchSQL(searchDsByType, ImmutableMap.of("type", type))
|
68
|
.stream()
|
69
|
.map(DatasourceFunctions::mapToSimpleDs)
|
70
|
.collect(Collectors.toList());
|
71
|
}
|
72
|
|
73
|
@Override
|
74
|
@Transactional(readOnly = true)
|
75
|
public List<? extends SearchApisEntry> searchApis(final String field, final Object value) throws DsmException {
|
76
|
try {
|
77
|
final StringWriter sql = new StringWriter();
|
78
|
|
79
|
if (field.equalsIgnoreCase("__search__")) {
|
80
|
sql.append(IOUtils.toString(searchApis.getInputStream()));
|
81
|
} else {
|
82
|
sql.append(IOUtils.toString(searchApisUsingField.getInputStream()));
|
83
|
sql.append(field);
|
84
|
sql.append("::text = ");
|
85
|
sql.append(":value");
|
86
|
}
|
87
|
|
88
|
return datasourceManagerClients
|
89
|
.searchSQL(sql.toString(), ImmutableMap.of("value", field.equalsIgnoreCase("__search__") ? "%" + value + "%" : value))
|
90
|
.stream()
|
91
|
.map(DatasourceFunctions::mapToSearchApisEntry)
|
92
|
.collect(Collectors.toList());
|
93
|
|
94
|
} catch (final Exception e) {
|
95
|
log.error("Error searching field " + field + " - value: " + value, e);
|
96
|
}
|
97
|
return new ArrayList<>();
|
98
|
}
|
99
|
|
100
|
@Override
|
101
|
@Transactional
|
102
|
public void saveDs(final Datasource<Organization<?>, Identity> ds) throws DsmException {
|
103
|
|
104
|
if (StringUtils.isBlank(ds.getAggregator())) {
|
105
|
ds.setAggregator("OPENAIRE");
|
106
|
}
|
107
|
|
108
|
ds.setManaged(true);
|
109
|
|
110
|
datasourceManagerClients.updateSQL(ds.getId(), addDs, AfterSqlUpdate.NONE, DatasourceFunctions.dsToMap(ds));
|
111
|
|
112
|
if (ds.getOrganizations() != null) {
|
113
|
for (final Organization<?> org : ds.getOrganizations()) {
|
114
|
final Map<String, Object> orgParams = DatasourceFunctions.orgToMap(ds.getId(), org);
|
115
|
datasourceManagerClients.updateSQL(ds.getId(), addOrg, AfterSqlUpdate.NONE, orgParams);
|
116
|
}
|
117
|
}
|
118
|
|
119
|
datasourceManagerClients.regenerateProfile(ds.getId());
|
120
|
}
|
121
|
|
122
|
@Override
|
123
|
@Transactional
|
124
|
public void deleteDs(final String dsId) throws DsmException {
|
125
|
datasourceManagerClients.updateSQL(fixDsId(dsId), deleteDs, AfterSqlUpdate.DELETE_DS_PROFILE, ImmutableMap.of("dsId", dsId));
|
126
|
}
|
127
|
|
128
|
@Override
|
129
|
public Datasource<Organization<?>, Identity> getDs(final String dsId) throws DsmException {
|
130
|
return datasourceManagerClients.getDatasourceById(fixDsId(dsId));
|
131
|
}
|
132
|
|
133
|
@Override
|
134
|
public void regenerateProfiles() throws DsmException {
|
135
|
datasourceManagerClients.regenerateProfiles();
|
136
|
}
|
137
|
|
138
|
@Override
|
139
|
public List<Api<ApiParam>> getApis(final String dsId) throws DsmException {
|
140
|
return datasourceManagerClients.getApis(fixDsId(dsId));
|
141
|
}
|
142
|
|
143
|
@Override
|
144
|
public void setManaged(final String dsId, final boolean managed) throws DsmException {
|
145
|
final String id = fixDsId(dsId);
|
146
|
|
147
|
final Map<String, Object> params = new HashMap<>();
|
148
|
params.put("managed", managed);
|
149
|
params.put("dsId", id);
|
150
|
|
151
|
datasourceManagerClients.updateSQL(id, setManaged, AfterSqlUpdate.UPDATE_DS_PROFILE, params);
|
152
|
|
153
|
}
|
154
|
|
155
|
@Override
|
156
|
public boolean isManaged(final String dsId) throws DsmException {
|
157
|
final String q = "SELECT * from dsm_datasources WHERE id = :dsId AND managed = true";
|
158
|
return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", fixDsId(dsId))).isEmpty();
|
159
|
}
|
160
|
|
161
|
@Override
|
162
|
@Transactional
|
163
|
public void setActive(final String dsId, final String apiId, final boolean active) throws DsmException {
|
164
|
final String id = fixDsId(dsId);
|
165
|
|
166
|
final Map<String, Object> params = new HashMap<>();
|
167
|
params.put("active", active);
|
168
|
params.put("apiId", apiId);
|
169
|
params.put("dsId", id);
|
170
|
|
171
|
datasourceManagerClients.updateSQL(id, setActive, AfterSqlUpdate.NONE, params);
|
172
|
if (!active) {
|
173
|
datasourceManagerClients.updateSQL(dsId, resetLastOperationsInfo, AfterSqlUpdate.NONE, params);
|
174
|
}
|
175
|
|
176
|
setManaged(id, true); // It also update the IS profile
|
177
|
}
|
178
|
|
179
|
@Override
|
180
|
@Transactional(readOnly = true)
|
181
|
public boolean isActive(final String dsId, final String apiId) throws DsmException {
|
182
|
final String q = "SELECT * from dsm_api WHERE id = :apiId AND datasource = :dsId AND active = true";
|
183
|
return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", fixDsId(dsId), "apiId", apiId)).isEmpty();
|
184
|
}
|
185
|
|
186
|
@Override
|
187
|
@Transactional(readOnly = true)
|
188
|
public boolean isRemovable(final String dsId, final String apiId) throws DsmException {
|
189
|
final String q = "SELECT * from dsm_api WHERE id = :apiId AND datasource = :dsId AND active != true AND removable = true";
|
190
|
return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", fixDsId(dsId), "apiId", apiId)).isEmpty();
|
191
|
}
|
192
|
|
193
|
@Override
|
194
|
@Transactional
|
195
|
public void updateCompliance(final String dsId, final String apiId, final String level, final boolean override) throws DsmException {
|
196
|
final String id = fixDsId(dsId);
|
197
|
|
198
|
if (!override) {
|
199
|
datasourceManagerClients.updateSQL(id, setCompliance, AfterSqlUpdate.NONE, ImmutableMap.of("level", level, "apiId", apiId, "dsId", id));
|
200
|
} else if (level != null) {
|
201
|
datasourceManagerClients.updateSQL(id, overrideCompliance, AfterSqlUpdate.NONE, ImmutableMap.of("level", level, "apiId", apiId, "dsId", id));
|
202
|
} else {
|
203
|
datasourceManagerClients.updateSQL(id, resetCompliance, AfterSqlUpdate.NONE, ImmutableMap.of("apiId", apiId, "dsId", id));
|
204
|
}
|
205
|
|
206
|
setManaged(id, true); // It also update the IS profile
|
207
|
}
|
208
|
|
209
|
@Override
|
210
|
|
211
|
public void setLastCollectionInfo(final String dsId, final String apiId, final String mdId, final Integer size, final Date date)
|
212
|
throws DsmException {
|
213
|
setLastOperationInfo(setLastCollectionInfo, fixDsId(dsId), apiId, mdId, size, date);
|
214
|
}
|
215
|
|
216
|
@Override
|
217
|
public void setLastAggregationInfo(final String dsId, final String apiId, final String mdId, final Integer size, final Date date)
|
218
|
throws DsmException {
|
219
|
setLastOperationInfo(setLastAggregationInfo, fixDsId(dsId), apiId, mdId, size, date);
|
220
|
}
|
221
|
|
222
|
@Override
|
223
|
public void setLastDownloadInfo(final String dsId, final String apiId, final String objId, final Integer size, final Date date)
|
224
|
throws DsmException {
|
225
|
setLastOperationInfo(setLastDownloadInfo, fixDsId(dsId), apiId, objId, size, date);
|
226
|
}
|
227
|
|
228
|
@Transactional
|
229
|
protected void setLastOperationInfo(final Resource sqlResource, final String dsId, final String apiId, final String mdId, final Integer size, final Date date)
|
230
|
throws DsmException {
|
231
|
final Map<String, Object> params = new HashMap<>();
|
232
|
params.put("dsId", dsId);
|
233
|
params.put("apiId", apiId);
|
234
|
params.put("mdId", mdId);
|
235
|
params.put("total", size);
|
236
|
if (date != null) {
|
237
|
params.put("date", new java.sql.Timestamp(date.getTime()));
|
238
|
}
|
239
|
|
240
|
datasourceManagerClients.updateSQL(dsId, sqlResource, AfterSqlUpdate.NONE, params);
|
241
|
|
242
|
setManaged(dsId, true); // It also update the IS profile
|
243
|
}
|
244
|
|
245
|
@Override
|
246
|
@Transactional
|
247
|
public void updateApiDetails(final String dsId,
|
248
|
final String apiId,
|
249
|
final String metadataIdentifierPath,
|
250
|
final String baseUrl,
|
251
|
final Map<String, String> params)
|
252
|
throws DsmException {
|
253
|
|
254
|
final String id = fixDsId(dsId);
|
255
|
|
256
|
// Delete old params
|
257
|
datasourceManagerClients.updateSQL(id, "DELETE FROM dsm_apiparams WHERE api = :api", AfterSqlUpdate.NONE, ImmutableMap.of("api", apiId));
|
258
|
|
259
|
// Insert new params
|
260
|
for (final Map.Entry<String, String> e : params.entrySet()) {
|
261
|
final Map<String, Object> sqlParams = ImmutableMap.of("param", e.getKey(), "value", e.getValue(), "api", apiId);
|
262
|
datasourceManagerClients.updateSQL(id, insertApiParam, AfterSqlUpdate.NONE, sqlParams);
|
263
|
}
|
264
|
|
265
|
// Update the BaseURL
|
266
|
datasourceManagerClients.updateSQL(id,
|
267
|
"UPDATE dsm_api SET baseurl = :baseurl WHERE id = :api",
|
268
|
AfterSqlUpdate.NONE,
|
269
|
ImmutableMap.of("baseurl", baseUrl, "api", apiId));
|
270
|
|
271
|
// Update the metadata_identifier_path
|
272
|
datasourceManagerClients.updateSQL(id,
|
273
|
"UPDATE dsm_api SET metadata_identifier_path = :path WHERE id = :api",
|
274
|
AfterSqlUpdate.NONE,
|
275
|
ImmutableMap.of("path", metadataIdentifierPath, "api", apiId));
|
276
|
|
277
|
setManaged(id, true); // It also update the IS profile
|
278
|
}
|
279
|
|
280
|
@Override
|
281
|
public List<? extends BrowsableField> listBrowsableFields() throws DsmException {
|
282
|
return getBrowsableFields();
|
283
|
}
|
284
|
|
285
|
@Override
|
286
|
public List<BrowseTerm> browseField(final String field) throws DsmException {
|
287
|
final Optional<DbBrowsableField> bf = getBrowsableFields()
|
288
|
.stream()
|
289
|
.filter(f -> f.getId().equals(field))
|
290
|
.findFirst();
|
291
|
|
292
|
if (bf.isPresent()) {
|
293
|
return datasourceManagerClients.searchSQL(bf.get().getSql(), new HashMap<>())
|
294
|
.stream()
|
295
|
.filter(m -> m.get("term") != null)
|
296
|
.filter(m -> m.get("count") != null)
|
297
|
.filter(m -> StringUtils.isNotBlank(m.get("term").toString()))
|
298
|
.map(m -> new BrowseTermImpl(m.get("term").toString(), NumberUtils.toInt(m.get("count").toString(), 0)))
|
299
|
.collect(Collectors.toList());
|
300
|
} else {
|
301
|
log.error("Not browsable field:" + field);
|
302
|
throw new DsmException("Not browsable field:" + field);
|
303
|
}
|
304
|
}
|
305
|
|
306
|
@Override
|
307
|
@Transactional
|
308
|
public void addApi(final Api<ApiParam> api) throws DsmException {
|
309
|
|
310
|
datasourceManagerClients.updateSQL(api.getDatasource(), insertApi, AfterSqlUpdate.NONE, DatasourceFunctions.apiToMap(api));
|
311
|
|
312
|
if (api.getApiParams() != null) {
|
313
|
api.getApiParams().forEach(p -> {
|
314
|
final ImmutableMap<String, Object> sqlParams = ImmutableMap.of("param", p.getParam(), "value", p.getValue(), "api", api.getId());
|
315
|
try {
|
316
|
datasourceManagerClients.updateSQL(api.getDatasource(), insertApiParam, AfterSqlUpdate.NONE, sqlParams);
|
317
|
} catch (final DsmException e) {
|
318
|
throw new RuntimeException(e);
|
319
|
}
|
320
|
});
|
321
|
}
|
322
|
|
323
|
setManaged(api.getDatasource(), true); // It also update the IS profile
|
324
|
}
|
325
|
|
326
|
@Override
|
327
|
@Transactional
|
328
|
public void deleteApi(final String dsId, final String apiId) throws DsmException {
|
329
|
if (!isRemovable(dsId, apiId)) { throw new DsmException("The api " + apiId + " can't be deleted"); }
|
330
|
|
331
|
datasourceManagerClients.updateSQL(dsId, "DELETE FROM dsm_apiparams WHERE api = :api", AfterSqlUpdate.NONE, ImmutableMap.of("api", apiId));
|
332
|
datasourceManagerClients.updateSQL(dsId, "DELETE FROM dsm_api WHERE id = :api", AfterSqlUpdate.NONE, ImmutableMap.of("api", apiId));
|
333
|
|
334
|
setManaged(dsId, true); // It also update the IS profile
|
335
|
}
|
336
|
|
337
|
public DatasourceManagerClients getDatasourceManagerClients() {
|
338
|
return datasourceManagerClients;
|
339
|
}
|
340
|
|
341
|
private String fixDsId(final String dsId) throws DsmException {
|
342
|
return dsId.endsWith(REPO_PROFILEID_SUFFIX) ? datasourceManagerClients.findDatasourceId(dsId) : dsId;
|
343
|
}
|
344
|
|
345
|
@Required
|
346
|
public void setDatasourceManagerClients(final DatasourceManagerClients datasourceManagerClients) {
|
347
|
this.datasourceManagerClients = datasourceManagerClients;
|
348
|
}
|
349
|
|
350
|
public List<DbBrowsableField> getBrowsableFields() {
|
351
|
return browsableFields;
|
352
|
}
|
353
|
|
354
|
@Required
|
355
|
public void setBrowsableFields(final List<DbBrowsableField> browsableFields) {
|
356
|
this.browsableFields = browsableFields;
|
357
|
}
|
358
|
|
359
|
}
|