Project

General

Profile

1 26600 sandro.lab
package eu.dnetlib.enabling.datasources;
2
3 50308 michele.ar
import java.io.StringWriter;
4 50596 claudio.at
import java.util.*;
5 49890 michele.ar
import java.util.stream.Collectors;
6 26600 sandro.lab
7 50596 claudio.at
import com.google.common.collect.ImmutableMap;
8
import eu.dnetlib.enabling.datasources.DatasourceManagerClients.AfterSqlUpdate;
9
import eu.dnetlib.enabling.datasources.common.*;
10 50308 michele.ar
import org.apache.commons.io.IOUtils;
11 50720 claudio.at
import org.apache.commons.lang3.StringUtils;
12
import org.apache.commons.lang3.math.NumberUtils;
13 33829 michele.ar
import org.apache.commons.logging.Log;
14
import org.apache.commons.logging.LogFactory;
15 26600 sandro.lab
import org.springframework.beans.factory.annotation.Required;
16 49890 michele.ar
import org.springframework.core.io.ClassPathResource;
17
import org.springframework.core.io.Resource;
18 50509 sandro.lab
import org.springframework.transaction.annotation.Transactional;
19 50435 michele.ar
20 50596 claudio.at
public class LocalOpenaireDatasourceManagerImpl implements LocalOpenaireDatasourceManager {
21 26600 sandro.lab
22
	private DatasourceManagerClients datasourceManagerClients;
23
24 33829 michele.ar
	private List<DbBrowsableField> browsableFields;
25
26 50345 michele.ar
	public static final String QUERY_BASEDIR = "/eu/dnetlib/enabling/datasources/queries/";
27
28 50278 michele.ar
	private static final Resource searchDsByType = new ClassPathResource(QUERY_BASEDIR + "searchDsByType.sql");
29 49928 michele.ar
	private static final Resource searchApis = new ClassPathResource(QUERY_BASEDIR + "searchApisNormal.sql");
30
	private static final Resource searchApisUsingField = new ClassPathResource(QUERY_BASEDIR + "searchApisUsingField.sql");
31
	private static final Resource addDs = new ClassPathResource(QUERY_BASEDIR + "addDatasource.sql");
32 49983 michele.ar
	private static final Resource addOrg = new ClassPathResource(QUERY_BASEDIR + "addOrganization.sql");
33 49928 michele.ar
	private static final Resource deleteDs = new ClassPathResource(QUERY_BASEDIR + "deleteDatasource.sql");
34
	private static final Resource setActive = new ClassPathResource(QUERY_BASEDIR + "setActive.sql");
35
	private static final Resource setManaged = new ClassPathResource(QUERY_BASEDIR + "setManaged.sql");
36
	private static final Resource setCompliance = new ClassPathResource(QUERY_BASEDIR + "setCompliance.sql");
37 49939 michele.ar
	private static final Resource overrideCompliance = new ClassPathResource(QUERY_BASEDIR + "overrideCompliance.sql");
38 50355 michele.ar
	private static final Resource resetCompliance = new ClassPathResource(QUERY_BASEDIR + "resetCompliance.sql");
39 49928 michele.ar
	private static final Resource setLastCollectionInfo = new ClassPathResource(QUERY_BASEDIR + "setLastCollectionInfo.sql");
40
	private static final Resource setLastAggregationInfo = new ClassPathResource(QUERY_BASEDIR + "setLastAggregationInfo.sql");
41
	private static final Resource setLastDownloadInfo = new ClassPathResource(QUERY_BASEDIR + "setLastDownloadInfo.sql");
42 54703 claudio.at
	private static final Resource setLastValidationJob = new ClassPathResource(QUERY_BASEDIR + "setLastValidationJob.sql");
43 50443 michele.ar
	private static final Resource resetLastOperationsInfo = new ClassPathResource(QUERY_BASEDIR + "resetLastOperationsInfo.sql");
44 49928 michele.ar
	private static final Resource insertApiParam = new ClassPathResource(QUERY_BASEDIR + "insertApiParam.sql");
45
	private static final Resource insertApi = new ClassPathResource(QUERY_BASEDIR + "insertApi.sql");
46 55619 alessia.ba
	private static final Resource setRemovable = new ClassPathResource(QUERY_BASEDIR + "setRemovable.sql");
47 49890 michele.ar
48 50596 claudio.at
	private static final Log log = LogFactory.getLog(LocalOpenaireDatasourceManagerImpl.class);
49 33829 michele.ar
50 50355 michele.ar
	private static final String REPO_PROFILEID_SUFFIX = "_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=";
51
52 26600 sandro.lab
	@Override
53 50509 sandro.lab
	@Transactional(readOnly = true)
54 50440 claudio.at
	public Set<String> listManagedDatasourceIds() throws DsmRuntimeException {
55
		try {
56
			return datasourceManagerClients.searchSQL("SELECT id FROM dsm_datasources WHERE managed = true", new HashMap<>())
57
					.stream()
58
					.map(m -> (String) m.get("id"))
59
					.collect(Collectors.toCollection(HashSet::new));
60 50441 michele.ar
		} catch (final DsmException e) {
61 50440 claudio.at
			throw new DsmRuntimeException(e);
62
		}
63
	}
64
65
	@Override
66 50509 sandro.lab
	@Transactional(readOnly = true)
67 50278 michele.ar
	public List<SimpleDatasource> searchDatasourcesByType(final String type) throws DsmException {
68 50509 sandro.lab
69 50278 michele.ar
		return datasourceManagerClients.searchSQL(searchDsByType, ImmutableMap.of("type", type))
70
				.stream()
71
				.map(DatasourceFunctions::mapToSimpleDs)
72
				.collect(Collectors.toList());
73
	}
74
75
	@Override
76 50509 sandro.lab
	@Transactional(readOnly = true)
77 50251 michele.ar
	public List<? extends SearchApisEntry> searchApis(final String field, final Object value) throws DsmException {
78 49890 michele.ar
		try {
79 50308 michele.ar
			final StringWriter sql = new StringWriter();
80 49970 michele.ar
81 50308 michele.ar
			if (field.equalsIgnoreCase("__search__")) {
82
				sql.append(IOUtils.toString(searchApis.getInputStream()));
83 49970 michele.ar
			} else {
84 50308 michele.ar
				sql.append(IOUtils.toString(searchApisUsingField.getInputStream()));
85
				sql.append(field);
86
				sql.append("::text = ");
87
				sql.append(":value");
88 49890 michele.ar
			}
89
90 50308 michele.ar
			return datasourceManagerClients
91
					.searchSQL(sql.toString(), ImmutableMap.of("value", field.equalsIgnoreCase("__search__") ? "%" + value + "%" : value))
92 49890 michele.ar
					.stream()
93
					.map(DatasourceFunctions::mapToSearchApisEntry)
94
					.collect(Collectors.toList());
95
96
		} catch (final Exception e) {
97
			log.error("Error searching field " + field + " - value: " + value, e);
98
		}
99
		return new ArrayList<>();
100
	}
101
102
	@Override
103 50509 sandro.lab
	@Transactional
104 50251 michele.ar
	public void saveDs(final Datasource<Organization<?>, Identity> ds) throws DsmException {
105 50435 michele.ar
106 38839 michele.ar
		if (StringUtils.isBlank(ds.getAggregator())) {
107
			ds.setAggregator("OPENAIRE");
108
		}
109
110 50435 michele.ar
		ds.setManaged(true);
111
112 50201 michele.ar
		datasourceManagerClients.updateSQL(ds.getId(), addDs, AfterSqlUpdate.NONE, DatasourceFunctions.dsToMap(ds));
113 38848 michele.ar
114 49983 michele.ar
		if (ds.getOrganizations() != null) {
115
			for (final Organization<?> org : ds.getOrganizations()) {
116 49990 michele.ar
				final Map<String, Object> orgParams = DatasourceFunctions.orgToMap(ds.getId(), org);
117 49983 michele.ar
				datasourceManagerClients.updateSQL(ds.getId(), addOrg, AfterSqlUpdate.NONE, orgParams);
118
			}
119 26600 sandro.lab
		}
120 55619 alessia.ba
		final Map<String, Object> params = new HashMap<>();
121
		params.put("removable", true);
122
		params.put("dsId", ds.getId());
123
		datasourceManagerClients.updateSQL(ds.getId(), setRemovable, AfterSqlUpdate.NONE, params);
124 49662 michele.ar
125 49983 michele.ar
		datasourceManagerClients.regenerateProfile(ds.getId());
126 26600 sandro.lab
	}
127
128
	@Override
129 50509 sandro.lab
	@Transactional
130 50251 michele.ar
	public void deleteDs(final String dsId) throws DsmException {
131 50355 michele.ar
		datasourceManagerClients.updateSQL(fixDsId(dsId), deleteDs, AfterSqlUpdate.DELETE_DS_PROFILE, ImmutableMap.of("dsId", dsId));
132 26600 sandro.lab
	}
133
134
	@Override
135 50251 michele.ar
	public Datasource<Organization<?>, Identity> getDs(final String dsId) throws DsmException {
136 50355 michele.ar
		return datasourceManagerClients.getDatasourceById(fixDsId(dsId));
137 26600 sandro.lab
	}
138
139
	@Override
140 50348 michele.ar
	public void regenerateProfiles() throws DsmException {
141
		datasourceManagerClients.regenerateProfiles();
142
	}
143
144
	@Override
145 50251 michele.ar
	public List<Api<ApiParam>> getApis(final String dsId) throws DsmException {
146 50355 michele.ar
		return datasourceManagerClients.getApis(fixDsId(dsId));
147 49928 michele.ar
	}
148
149
	@Override
150 50251 michele.ar
	public void setManaged(final String dsId, final boolean managed) throws DsmException {
151 50355 michele.ar
		final String id = fixDsId(dsId);
152
153 49928 michele.ar
		final Map<String, Object> params = new HashMap<>();
154
		params.put("managed", managed);
155 50355 michele.ar
		params.put("dsId", id);
156 49928 michele.ar
157 55619 alessia.ba
		datasourceManagerClients.updateSQL(id, setManaged, AfterSqlUpdate.NONE, params);
158 49928 michele.ar
159 55619 alessia.ba
		params.clear();
160
		params.put("removable", true);
161
		params.put("dsId", id);
162
163
		datasourceManagerClients.updateSQL(id, setRemovable, AfterSqlUpdate.UPDATE_DS_PROFILE, params);
164
165 49928 michele.ar
	}
166
167
	@Override
168 50251 michele.ar
	public boolean isManaged(final String dsId) throws DsmException {
169 49928 michele.ar
		final String q = "SELECT * from dsm_datasources WHERE id = :dsId AND managed = true";
170 50355 michele.ar
		return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", fixDsId(dsId))).isEmpty();
171 49928 michele.ar
	}
172
173
	@Override
174 50509 sandro.lab
	@Transactional
175 50251 michele.ar
	public void setActive(final String dsId, final String apiId, final boolean active) throws DsmException {
176 50355 michele.ar
		final String id = fixDsId(dsId);
177
178 49890 michele.ar
		final Map<String, Object> params = new HashMap<>();
179
		params.put("active", active);
180
		params.put("apiId", apiId);
181 50355 michele.ar
		params.put("dsId", id);
182 26600 sandro.lab
183 50435 michele.ar
		datasourceManagerClients.updateSQL(id, setActive, AfterSqlUpdate.NONE, params);
184 49890 michele.ar
		if (!active) {
185 50443 michele.ar
			datasourceManagerClients.updateSQL(dsId, resetLastOperationsInfo, AfterSqlUpdate.NONE, params);
186 49890 michele.ar
		}
187 50435 michele.ar
188
		setManaged(id, true); // It also update the IS profile
189 49890 michele.ar
	}
190 26600 sandro.lab
191 49890 michele.ar
	@Override
192 50509 sandro.lab
	@Transactional(readOnly = true)
193 50251 michele.ar
	public boolean isActive(final String dsId, final String apiId) throws DsmException {
194 49928 michele.ar
		final String q = "SELECT * from dsm_api WHERE id = :apiId AND datasource = :dsId AND active = true";
195 50355 michele.ar
		return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", fixDsId(dsId), "apiId", apiId)).isEmpty();
196 26600 sandro.lab
	}
197
198
	@Override
199 50509 sandro.lab
	@Transactional(readOnly = true)
200 50251 michele.ar
	public boolean isRemovable(final String dsId, final String apiId) throws DsmException {
201 49970 michele.ar
		final String q = "SELECT * from dsm_api WHERE id = :apiId AND datasource = :dsId AND active != true AND removable = true";
202 50355 michele.ar
		return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", fixDsId(dsId), "apiId", apiId)).isEmpty();
203 49970 michele.ar
	}
204
205
	@Override
206 50509 sandro.lab
	@Transactional
207 50251 michele.ar
	public void updateCompliance(final String dsId, final String apiId, final String level, final boolean override) throws DsmException {
208 50355 michele.ar
		final String id = fixDsId(dsId);
209
210
		if (!override) {
211 50435 michele.ar
			datasourceManagerClients.updateSQL(id, setCompliance, AfterSqlUpdate.NONE, ImmutableMap.of("level", level, "apiId", apiId, "dsId", id));
212 50355 michele.ar
		} else if (level != null) {
213 50435 michele.ar
			datasourceManagerClients.updateSQL(id, overrideCompliance, AfterSqlUpdate.NONE, ImmutableMap.of("level", level, "apiId", apiId, "dsId", id));
214 50355 michele.ar
		} else {
215 50435 michele.ar
			datasourceManagerClients.updateSQL(id, resetCompliance, AfterSqlUpdate.NONE, ImmutableMap.of("apiId", apiId, "dsId", id));
216 50355 michele.ar
		}
217 50435 michele.ar
218
		setManaged(id, true); // It also update the IS profile
219 26600 sandro.lab
	}
220
221
	@Override
222 50509 sandro.lab
223 49928 michele.ar
	public void setLastCollectionInfo(final String dsId, final String apiId, final String mdId, final Integer size, final Date date)
224 50251 michele.ar
			throws DsmException {
225 50355 michele.ar
		setLastOperationInfo(setLastCollectionInfo, fixDsId(dsId), apiId, mdId, size, date);
226 26600 sandro.lab
	}
227
228
	@Override
229 49928 michele.ar
	public void setLastAggregationInfo(final String dsId, final String apiId, final String mdId, final Integer size, final Date date)
230 50251 michele.ar
			throws DsmException {
231 50355 michele.ar
		setLastOperationInfo(setLastAggregationInfo, fixDsId(dsId), apiId, mdId, size, date);
232 26600 sandro.lab
	}
233
234
	@Override
235 49928 michele.ar
	public void setLastDownloadInfo(final String dsId, final String apiId, final String objId, final Integer size, final Date date)
236 50251 michele.ar
			throws DsmException {
237 50355 michele.ar
		setLastOperationInfo(setLastDownloadInfo, fixDsId(dsId), apiId, objId, size, date);
238 49928 michele.ar
	}
239 26600 sandro.lab
240 54703 claudio.at
	@Override
241
	public void setLastValidationJob(String dsId, String apiId, String jobId) throws DsmException {
242
		final Map<String, Object> params = new HashMap<>();
243
		params.put("dsId", dsId);
244
		params.put("apiId", apiId);
245
		params.put("jobId", jobId);
246
247
		datasourceManagerClients.updateSQL(dsId, setLastValidationJob, AfterSqlUpdate.NONE, params);
248
		setManaged(dsId, true); // It also update the IS profile
249
	}
250
251 50509 sandro.lab
	@Transactional
252
	protected void setLastOperationInfo(final Resource sqlResource, final String dsId, final String apiId, final String mdId, final Integer size, final Date date)
253 50251 michele.ar
			throws DsmException {
254 49928 michele.ar
		final Map<String, Object> params = new HashMap<>();
255
		params.put("dsId", dsId);
256
		params.put("apiId", apiId);
257
		params.put("mdId", mdId);
258 50363 michele.ar
		params.put("total", size);
259 50440 claudio.at
		if (date != null) {
260
			params.put("date", new java.sql.Timestamp(date.getTime()));
261
		}
262 50435 michele.ar
263
		datasourceManagerClients.updateSQL(dsId, sqlResource, AfterSqlUpdate.NONE, params);
264
265
		setManaged(dsId, true); // It also update the IS profile
266 26600 sandro.lab
	}
267
268
	@Override
269 50509 sandro.lab
	@Transactional
270 50278 michele.ar
	public void updateApiDetails(final String dsId,
271
			final String apiId,
272
			final String metadataIdentifierPath,
273
			final String baseUrl,
274
			final Map<String, String> params)
275 50251 michele.ar
			throws DsmException {
276 50355 michele.ar
277
		final String id = fixDsId(dsId);
278
279 49928 michele.ar
		// Delete old params
280 50355 michele.ar
		datasourceManagerClients.updateSQL(id, "DELETE FROM dsm_apiparams WHERE api = :api", AfterSqlUpdate.NONE, ImmutableMap.of("api", apiId));
281 26600 sandro.lab
282 49928 michele.ar
		// Insert new params
283
		for (final Map.Entry<String, String> e : params.entrySet()) {
284
			final Map<String, Object> sqlParams = ImmutableMap.of("param", e.getKey(), "value", e.getValue(), "api", apiId);
285 50355 michele.ar
			datasourceManagerClients.updateSQL(id, insertApiParam, AfterSqlUpdate.NONE, sqlParams);
286 26600 sandro.lab
		}
287
288 50278 michele.ar
		// Update the BaseURL
289 50355 michele.ar
		datasourceManagerClients.updateSQL(id,
290 50278 michele.ar
				"UPDATE dsm_api SET baseurl = :baseurl WHERE id = :api",
291
				AfterSqlUpdate.NONE,
292 49928 michele.ar
				ImmutableMap.of("baseurl", baseUrl, "api", apiId));
293 50278 michele.ar
294
		// Update the metadata_identifier_path
295 50355 michele.ar
		datasourceManagerClients.updateSQL(id,
296 50278 michele.ar
				"UPDATE dsm_api SET metadata_identifier_path = :path WHERE id = :api",
297
				AfterSqlUpdate.NONE,
298
				ImmutableMap.of("path", metadataIdentifierPath, "api", apiId));
299
300 50435 michele.ar
		setManaged(id, true); // It also update the IS profile
301 26600 sandro.lab
	}
302
303
	@Override
304 50251 michele.ar
	public List<? extends BrowsableField> listBrowsableFields() throws DsmException {
305 49928 michele.ar
		return getBrowsableFields();
306 26600 sandro.lab
	}
307
308
	@Override
309 50251 michele.ar
	public List<BrowseTerm> browseField(final String field) throws DsmException {
310 50284 michele.ar
		final Optional<DbBrowsableField> bf = getBrowsableFields()
311 49928 michele.ar
				.stream()
312
				.filter(f -> f.getId().equals(field))
313 50284 michele.ar
				.findFirst();
314
315
		if (bf.isPresent()) {
316
			return datasourceManagerClients.searchSQL(bf.get().getSql(), new HashMap<>())
317
					.stream()
318
					.filter(m -> m.get("term") != null)
319
					.filter(m -> m.get("count") != null)
320
					.filter(m -> StringUtils.isNotBlank(m.get("term").toString()))
321
					.map(m -> new BrowseTermImpl(m.get("term").toString(), NumberUtils.toInt(m.get("count").toString(), 0)))
322
					.collect(Collectors.toList());
323
		} else {
324
			log.error("Not browsable field:" + field);
325
			throw new DsmException("Not browsable field:" + field);
326
		}
327 33829 michele.ar
	}
328
329
	@Override
330 50509 sandro.lab
	@Transactional
331 50251 michele.ar
	public void addApi(final Api<ApiParam> api) throws DsmException {
332 33829 michele.ar
333 49928 michele.ar
		datasourceManagerClients.updateSQL(api.getDatasource(), insertApi, AfterSqlUpdate.NONE, DatasourceFunctions.apiToMap(api));
334 33829 michele.ar
335 50909 claudio.at
		if (api.getApiParams() != null) {
336
			api.getApiParams().forEach(p -> {
337
				final ImmutableMap<String, Object> sqlParams = ImmutableMap.of("param", p.getParam(), "value", p.getValue(), "api", api.getId());
338
				try {
339
					datasourceManagerClients.updateSQL(api.getDatasource(), insertApiParam, AfterSqlUpdate.NONE, sqlParams);
340
				} catch (final DsmException e) {
341
					throw new RuntimeException(e);
342
				}
343
			});
344
		}
345 33829 michele.ar
346 50435 michele.ar
		setManaged(api.getDatasource(), true); // It also update the IS profile
347 33829 michele.ar
	}
348
349 49928 michele.ar
	@Override
350 50509 sandro.lab
	@Transactional
351 50251 michele.ar
	public void deleteApi(final String dsId, final String apiId) throws DsmException {
352
		if (!isRemovable(dsId, apiId)) { throw new DsmException("The api " + apiId + " can't be deleted"); }
353 33829 michele.ar
354 49928 michele.ar
		datasourceManagerClients.updateSQL(dsId, "DELETE FROM dsm_apiparams WHERE api = :api", AfterSqlUpdate.NONE, ImmutableMap.of("api", apiId));
355 50435 michele.ar
		datasourceManagerClients.updateSQL(dsId, "DELETE FROM dsm_api WHERE id = :api", AfterSqlUpdate.NONE, ImmutableMap.of("api", apiId));
356
357
		setManaged(dsId, true); // It also update the IS profile
358 33829 michele.ar
	}
359
360
	public DatasourceManagerClients getDatasourceManagerClients() {
361
		return datasourceManagerClients;
362
	}
363
364 50355 michele.ar
	private String fixDsId(final String dsId) throws DsmException {
365
		return dsId.endsWith(REPO_PROFILEID_SUFFIX) ? datasourceManagerClients.findDatasourceId(dsId) : dsId;
366
	}
367
368 26600 sandro.lab
	@Required
369
	public void setDatasourceManagerClients(final DatasourceManagerClients datasourceManagerClients) {
370
		this.datasourceManagerClients = datasourceManagerClients;
371
	}
372
373 33829 michele.ar
	public List<DbBrowsableField> getBrowsableFields() {
374
		return browsableFields;
375
	}
376
377
	@Required
378
	public void setBrowsableFields(final List<DbBrowsableField> browsableFields) {
379
		this.browsableFields = browsableFields;
380
	}
381 49890 michele.ar
382 26600 sandro.lab
}