Project

General

Profile

1 26600 sandro.lab
package eu.dnetlib.enabling.datasources;
2
3 50308 michele.ar
import java.io.StringWriter;
4 50596 claudio.at
import java.util.*;
5 49890 michele.ar
import java.util.stream.Collectors;
6 26600 sandro.lab
7 50596 claudio.at
import com.google.common.collect.ImmutableMap;
8
import eu.dnetlib.enabling.datasources.DatasourceManagerClients.AfterSqlUpdate;
9
import eu.dnetlib.enabling.datasources.common.*;
10 50308 michele.ar
import org.apache.commons.io.IOUtils;
11 50720 claudio.at
import org.apache.commons.lang3.StringUtils;
12
import org.apache.commons.lang3.math.NumberUtils;
13 33829 michele.ar
import org.apache.commons.logging.Log;
14
import org.apache.commons.logging.LogFactory;
15 26600 sandro.lab
import org.springframework.beans.factory.annotation.Required;
16 49890 michele.ar
import org.springframework.core.io.ClassPathResource;
17
import org.springframework.core.io.Resource;
18 50509 sandro.lab
import org.springframework.transaction.annotation.Transactional;
19 50435 michele.ar
20 50596 claudio.at
public class LocalOpenaireDatasourceManagerImpl implements LocalOpenaireDatasourceManager {
21 26600 sandro.lab
22
	private DatasourceManagerClients datasourceManagerClients;
23
24 33829 michele.ar
	private List<DbBrowsableField> browsableFields;
25
26 50345 michele.ar
	public static final String QUERY_BASEDIR = "/eu/dnetlib/enabling/datasources/queries/";
27
28 50278 michele.ar
	private static final Resource searchDsByType = new ClassPathResource(QUERY_BASEDIR + "searchDsByType.sql");
29 49928 michele.ar
	private static final Resource searchApis = new ClassPathResource(QUERY_BASEDIR + "searchApisNormal.sql");
30
	private static final Resource searchApisUsingField = new ClassPathResource(QUERY_BASEDIR + "searchApisUsingField.sql");
31
	private static final Resource addDs = new ClassPathResource(QUERY_BASEDIR + "addDatasource.sql");
32 49983 michele.ar
	private static final Resource addOrg = new ClassPathResource(QUERY_BASEDIR + "addOrganization.sql");
33 49928 michele.ar
	private static final Resource deleteDs = new ClassPathResource(QUERY_BASEDIR + "deleteDatasource.sql");
34
	private static final Resource setActive = new ClassPathResource(QUERY_BASEDIR + "setActive.sql");
35
	private static final Resource setManaged = new ClassPathResource(QUERY_BASEDIR + "setManaged.sql");
36
	private static final Resource setCompliance = new ClassPathResource(QUERY_BASEDIR + "setCompliance.sql");
37 49939 michele.ar
	private static final Resource overrideCompliance = new ClassPathResource(QUERY_BASEDIR + "overrideCompliance.sql");
38 50355 michele.ar
	private static final Resource resetCompliance = new ClassPathResource(QUERY_BASEDIR + "resetCompliance.sql");
39 49928 michele.ar
	private static final Resource setLastCollectionInfo = new ClassPathResource(QUERY_BASEDIR + "setLastCollectionInfo.sql");
40
	private static final Resource setLastAggregationInfo = new ClassPathResource(QUERY_BASEDIR + "setLastAggregationInfo.sql");
41
	private static final Resource setLastDownloadInfo = new ClassPathResource(QUERY_BASEDIR + "setLastDownloadInfo.sql");
42 50443 michele.ar
	private static final Resource resetLastOperationsInfo = new ClassPathResource(QUERY_BASEDIR + "resetLastOperationsInfo.sql");
43 49928 michele.ar
	private static final Resource insertApiParam = new ClassPathResource(QUERY_BASEDIR + "insertApiParam.sql");
44
	private static final Resource insertApi = new ClassPathResource(QUERY_BASEDIR + "insertApi.sql");
45 49890 michele.ar
46 50596 claudio.at
	private static final Log log = LogFactory.getLog(LocalOpenaireDatasourceManagerImpl.class);
47 33829 michele.ar
48 50355 michele.ar
	private static final String REPO_PROFILEID_SUFFIX = "_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=";
49
50 26600 sandro.lab
	@Override
51 50509 sandro.lab
	@Transactional(readOnly = true)
52 50440 claudio.at
	public Set<String> listManagedDatasourceIds() throws DsmRuntimeException {
53
		try {
54
			return datasourceManagerClients.searchSQL("SELECT id FROM dsm_datasources WHERE managed = true", new HashMap<>())
55
					.stream()
56
					.map(m -> (String) m.get("id"))
57
					.collect(Collectors.toCollection(HashSet::new));
58 50441 michele.ar
		} catch (final DsmException e) {
59 50440 claudio.at
			throw new DsmRuntimeException(e);
60
		}
61
	}
62
63
	@Override
64 50509 sandro.lab
	@Transactional(readOnly = true)
65 50278 michele.ar
	public List<SimpleDatasource> searchDatasourcesByType(final String type) throws DsmException {
66 50509 sandro.lab
67 50278 michele.ar
		return datasourceManagerClients.searchSQL(searchDsByType, ImmutableMap.of("type", type))
68
				.stream()
69
				.map(DatasourceFunctions::mapToSimpleDs)
70
				.collect(Collectors.toList());
71
	}
72
73
	@Override
74 50509 sandro.lab
	@Transactional(readOnly = true)
75 50251 michele.ar
	public List<? extends SearchApisEntry> searchApis(final String field, final Object value) throws DsmException {
76 49890 michele.ar
		try {
77 50308 michele.ar
			final StringWriter sql = new StringWriter();
78 49970 michele.ar
79 50308 michele.ar
			if (field.equalsIgnoreCase("__search__")) {
80
				sql.append(IOUtils.toString(searchApis.getInputStream()));
81 49970 michele.ar
			} else {
82 50308 michele.ar
				sql.append(IOUtils.toString(searchApisUsingField.getInputStream()));
83
				sql.append(field);
84
				sql.append("::text = ");
85
				sql.append(":value");
86 49890 michele.ar
			}
87
88 50308 michele.ar
			return datasourceManagerClients
89
					.searchSQL(sql.toString(), ImmutableMap.of("value", field.equalsIgnoreCase("__search__") ? "%" + value + "%" : value))
90 49890 michele.ar
					.stream()
91
					.map(DatasourceFunctions::mapToSearchApisEntry)
92
					.collect(Collectors.toList());
93
94
		} catch (final Exception e) {
95
			log.error("Error searching field " + field + " - value: " + value, e);
96
		}
97
		return new ArrayList<>();
98
	}
99
100
	@Override
101 50509 sandro.lab
	@Transactional
102 50251 michele.ar
	public void saveDs(final Datasource<Organization<?>, Identity> ds) throws DsmException {
103 50435 michele.ar
104 38839 michele.ar
		if (StringUtils.isBlank(ds.getAggregator())) {
105
			ds.setAggregator("OPENAIRE");
106
		}
107
108 50435 michele.ar
		ds.setManaged(true);
109
110 50201 michele.ar
		datasourceManagerClients.updateSQL(ds.getId(), addDs, AfterSqlUpdate.NONE, DatasourceFunctions.dsToMap(ds));
111 38848 michele.ar
112 49983 michele.ar
		if (ds.getOrganizations() != null) {
113
			for (final Organization<?> org : ds.getOrganizations()) {
114 49990 michele.ar
				final Map<String, Object> orgParams = DatasourceFunctions.orgToMap(ds.getId(), org);
115 49983 michele.ar
				datasourceManagerClients.updateSQL(ds.getId(), addOrg, AfterSqlUpdate.NONE, orgParams);
116
			}
117 26600 sandro.lab
		}
118 49662 michele.ar
119 49983 michele.ar
		datasourceManagerClients.regenerateProfile(ds.getId());
120 26600 sandro.lab
	}
121
122
	@Override
123 50509 sandro.lab
	@Transactional
124 50251 michele.ar
	public void deleteDs(final String dsId) throws DsmException {
125 50355 michele.ar
		datasourceManagerClients.updateSQL(fixDsId(dsId), deleteDs, AfterSqlUpdate.DELETE_DS_PROFILE, ImmutableMap.of("dsId", dsId));
126 26600 sandro.lab
	}
127
128
	@Override
129 50251 michele.ar
	public Datasource<Organization<?>, Identity> getDs(final String dsId) throws DsmException {
130 50355 michele.ar
		return datasourceManagerClients.getDatasourceById(fixDsId(dsId));
131 26600 sandro.lab
	}
132
133
	@Override
134 50348 michele.ar
	public void regenerateProfiles() throws DsmException {
135
		datasourceManagerClients.regenerateProfiles();
136
	}
137
138
	@Override
139 50251 michele.ar
	public List<Api<ApiParam>> getApis(final String dsId) throws DsmException {
140 50355 michele.ar
		return datasourceManagerClients.getApis(fixDsId(dsId));
141 49928 michele.ar
	}
142
143
	@Override
144 50251 michele.ar
	public void setManaged(final String dsId, final boolean managed) throws DsmException {
145 50355 michele.ar
		final String id = fixDsId(dsId);
146
147 49928 michele.ar
		final Map<String, Object> params = new HashMap<>();
148
		params.put("managed", managed);
149 50355 michele.ar
		params.put("dsId", id);
150 49928 michele.ar
151 50355 michele.ar
		datasourceManagerClients.updateSQL(id, setManaged, AfterSqlUpdate.UPDATE_DS_PROFILE, params);
152 49928 michele.ar
153
	}
154
155
	@Override
156 50251 michele.ar
	public boolean isManaged(final String dsId) throws DsmException {
157 49928 michele.ar
		final String q = "SELECT * from dsm_datasources WHERE id = :dsId AND managed = true";
158 50355 michele.ar
		return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", fixDsId(dsId))).isEmpty();
159 49928 michele.ar
	}
160
161
	@Override
162 50509 sandro.lab
	@Transactional
163 50251 michele.ar
	public void setActive(final String dsId, final String apiId, final boolean active) throws DsmException {
164 50355 michele.ar
		final String id = fixDsId(dsId);
165
166 49890 michele.ar
		final Map<String, Object> params = new HashMap<>();
167
		params.put("active", active);
168
		params.put("apiId", apiId);
169 50355 michele.ar
		params.put("dsId", id);
170 26600 sandro.lab
171 50435 michele.ar
		datasourceManagerClients.updateSQL(id, setActive, AfterSqlUpdate.NONE, params);
172 49890 michele.ar
		if (!active) {
173 50443 michele.ar
			datasourceManagerClients.updateSQL(dsId, resetLastOperationsInfo, AfterSqlUpdate.NONE, params);
174 49890 michele.ar
		}
175 50435 michele.ar
176
		setManaged(id, true); // It also update the IS profile
177 49890 michele.ar
	}
178 26600 sandro.lab
179 49890 michele.ar
	@Override
180 50509 sandro.lab
	@Transactional(readOnly = true)
181 50251 michele.ar
	public boolean isActive(final String dsId, final String apiId) throws DsmException {
182 49928 michele.ar
		final String q = "SELECT * from dsm_api WHERE id = :apiId AND datasource = :dsId AND active = true";
183 50355 michele.ar
		return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", fixDsId(dsId), "apiId", apiId)).isEmpty();
184 26600 sandro.lab
	}
185
186
	@Override
187 50509 sandro.lab
	@Transactional(readOnly = true)
188 50251 michele.ar
	public boolean isRemovable(final String dsId, final String apiId) throws DsmException {
189 49970 michele.ar
		final String q = "SELECT * from dsm_api WHERE id = :apiId AND datasource = :dsId AND active != true AND removable = true";
190 50355 michele.ar
		return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", fixDsId(dsId), "apiId", apiId)).isEmpty();
191 49970 michele.ar
	}
192
193
	@Override
194 50509 sandro.lab
	@Transactional
195 50251 michele.ar
	public void updateCompliance(final String dsId, final String apiId, final String level, final boolean override) throws DsmException {
196 50355 michele.ar
		final String id = fixDsId(dsId);
197
198
		if (!override) {
199 50435 michele.ar
			datasourceManagerClients.updateSQL(id, setCompliance, AfterSqlUpdate.NONE, ImmutableMap.of("level", level, "apiId", apiId, "dsId", id));
200 50355 michele.ar
		} else if (level != null) {
201 50435 michele.ar
			datasourceManagerClients.updateSQL(id, overrideCompliance, AfterSqlUpdate.NONE, ImmutableMap.of("level", level, "apiId", apiId, "dsId", id));
202 50355 michele.ar
		} else {
203 50435 michele.ar
			datasourceManagerClients.updateSQL(id, resetCompliance, AfterSqlUpdate.NONE, ImmutableMap.of("apiId", apiId, "dsId", id));
204 50355 michele.ar
		}
205 50435 michele.ar
206
		setManaged(id, true); // It also update the IS profile
207 26600 sandro.lab
	}
208
209
	@Override
210 50509 sandro.lab
211 49928 michele.ar
	public void setLastCollectionInfo(final String dsId, final String apiId, final String mdId, final Integer size, final Date date)
212 50251 michele.ar
			throws DsmException {
213 50355 michele.ar
		setLastOperationInfo(setLastCollectionInfo, fixDsId(dsId), apiId, mdId, size, date);
214 26600 sandro.lab
	}
215
216
	@Override
217 49928 michele.ar
	public void setLastAggregationInfo(final String dsId, final String apiId, final String mdId, final Integer size, final Date date)
218 50251 michele.ar
			throws DsmException {
219 50355 michele.ar
		setLastOperationInfo(setLastAggregationInfo, fixDsId(dsId), apiId, mdId, size, date);
220 26600 sandro.lab
	}
221
222
	@Override
223 49928 michele.ar
	public void setLastDownloadInfo(final String dsId, final String apiId, final String objId, final Integer size, final Date date)
224 50251 michele.ar
			throws DsmException {
225 50355 michele.ar
		setLastOperationInfo(setLastDownloadInfo, fixDsId(dsId), apiId, objId, size, date);
226 49928 michele.ar
	}
227 26600 sandro.lab
228 50509 sandro.lab
	@Transactional
229
	protected void setLastOperationInfo(final Resource sqlResource, final String dsId, final String apiId, final String mdId, final Integer size, final Date date)
230 50251 michele.ar
			throws DsmException {
231 49928 michele.ar
		final Map<String, Object> params = new HashMap<>();
232
		params.put("dsId", dsId);
233
		params.put("apiId", apiId);
234
		params.put("mdId", mdId);
235 50363 michele.ar
		params.put("total", size);
236 50440 claudio.at
		if (date != null) {
237
			params.put("date", new java.sql.Timestamp(date.getTime()));
238
		}
239 50435 michele.ar
240
		datasourceManagerClients.updateSQL(dsId, sqlResource, AfterSqlUpdate.NONE, params);
241
242
		setManaged(dsId, true); // It also update the IS profile
243 26600 sandro.lab
	}
244
245
	@Override
246 50509 sandro.lab
	@Transactional
247 50278 michele.ar
	public void updateApiDetails(final String dsId,
248
			final String apiId,
249
			final String metadataIdentifierPath,
250
			final String baseUrl,
251
			final Map<String, String> params)
252 50251 michele.ar
			throws DsmException {
253 50355 michele.ar
254
		final String id = fixDsId(dsId);
255
256 49928 michele.ar
		// Delete old params
257 50355 michele.ar
		datasourceManagerClients.updateSQL(id, "DELETE FROM dsm_apiparams WHERE api = :api", AfterSqlUpdate.NONE, ImmutableMap.of("api", apiId));
258 26600 sandro.lab
259 49928 michele.ar
		// Insert new params
260
		for (final Map.Entry<String, String> e : params.entrySet()) {
261
			final Map<String, Object> sqlParams = ImmutableMap.of("param", e.getKey(), "value", e.getValue(), "api", apiId);
262 50355 michele.ar
			datasourceManagerClients.updateSQL(id, insertApiParam, AfterSqlUpdate.NONE, sqlParams);
263 26600 sandro.lab
		}
264
265 50278 michele.ar
		// Update the BaseURL
266 50355 michele.ar
		datasourceManagerClients.updateSQL(id,
267 50278 michele.ar
				"UPDATE dsm_api SET baseurl = :baseurl WHERE id = :api",
268
				AfterSqlUpdate.NONE,
269 49928 michele.ar
				ImmutableMap.of("baseurl", baseUrl, "api", apiId));
270 50278 michele.ar
271
		// Update the metadata_identifier_path
272 50355 michele.ar
		datasourceManagerClients.updateSQL(id,
273 50278 michele.ar
				"UPDATE dsm_api SET metadata_identifier_path = :path WHERE id = :api",
274
				AfterSqlUpdate.NONE,
275
				ImmutableMap.of("path", metadataIdentifierPath, "api", apiId));
276
277 50435 michele.ar
		setManaged(id, true); // It also update the IS profile
278 26600 sandro.lab
	}
279
280
	@Override
281 50251 michele.ar
	public List<? extends BrowsableField> listBrowsableFields() throws DsmException {
282 49928 michele.ar
		return getBrowsableFields();
283 26600 sandro.lab
	}
284
285
	@Override
286 50251 michele.ar
	public List<BrowseTerm> browseField(final String field) throws DsmException {
287 50284 michele.ar
		final Optional<DbBrowsableField> bf = getBrowsableFields()
288 49928 michele.ar
				.stream()
289
				.filter(f -> f.getId().equals(field))
290 50284 michele.ar
				.findFirst();
291
292
		if (bf.isPresent()) {
293
			return datasourceManagerClients.searchSQL(bf.get().getSql(), new HashMap<>())
294
					.stream()
295
					.filter(m -> m.get("term") != null)
296
					.filter(m -> m.get("count") != null)
297
					.filter(m -> StringUtils.isNotBlank(m.get("term").toString()))
298
					.map(m -> new BrowseTermImpl(m.get("term").toString(), NumberUtils.toInt(m.get("count").toString(), 0)))
299
					.collect(Collectors.toList());
300
		} else {
301
			log.error("Not browsable field:" + field);
302
			throw new DsmException("Not browsable field:" + field);
303
		}
304 33829 michele.ar
	}
305
306
	@Override
307 50509 sandro.lab
	@Transactional
308 50251 michele.ar
	public void addApi(final Api<ApiParam> api) throws DsmException {
309 33829 michele.ar
310 49928 michele.ar
		datasourceManagerClients.updateSQL(api.getDatasource(), insertApi, AfterSqlUpdate.NONE, DatasourceFunctions.apiToMap(api));
311 33829 michele.ar
312 50909 claudio.at
		if (api.getApiParams() != null) {
313
			api.getApiParams().forEach(p -> {
314
				final ImmutableMap<String, Object> sqlParams = ImmutableMap.of("param", p.getParam(), "value", p.getValue(), "api", api.getId());
315
				try {
316
					datasourceManagerClients.updateSQL(api.getDatasource(), insertApiParam, AfterSqlUpdate.NONE, sqlParams);
317
				} catch (final DsmException e) {
318
					throw new RuntimeException(e);
319
				}
320
			});
321
		}
322 33829 michele.ar
323 50435 michele.ar
		setManaged(api.getDatasource(), true); // It also update the IS profile
324 33829 michele.ar
	}
325
326 49928 michele.ar
	@Override
327 50509 sandro.lab
	@Transactional
328 50251 michele.ar
	public void deleteApi(final String dsId, final String apiId) throws DsmException {
329
		if (!isRemovable(dsId, apiId)) { throw new DsmException("The api " + apiId + " can't be deleted"); }
330 33829 michele.ar
331 49928 michele.ar
		datasourceManagerClients.updateSQL(dsId, "DELETE FROM dsm_apiparams WHERE api = :api", AfterSqlUpdate.NONE, ImmutableMap.of("api", apiId));
332 50435 michele.ar
		datasourceManagerClients.updateSQL(dsId, "DELETE FROM dsm_api WHERE id = :api", AfterSqlUpdate.NONE, ImmutableMap.of("api", apiId));
333
334
		setManaged(dsId, true); // It also update the IS profile
335 33829 michele.ar
	}
336
337
	public DatasourceManagerClients getDatasourceManagerClients() {
338
		return datasourceManagerClients;
339
	}
340
341 50355 michele.ar
	private String fixDsId(final String dsId) throws DsmException {
342
		return dsId.endsWith(REPO_PROFILEID_SUFFIX) ? datasourceManagerClients.findDatasourceId(dsId) : dsId;
343
	}
344
345 26600 sandro.lab
	@Required
346
	public void setDatasourceManagerClients(final DatasourceManagerClients datasourceManagerClients) {
347
		this.datasourceManagerClients = datasourceManagerClients;
348
	}
349
350 33829 michele.ar
	public List<DbBrowsableField> getBrowsableFields() {
351
		return browsableFields;
352
	}
353
354
	@Required
355
	public void setBrowsableFields(final List<DbBrowsableField> browsableFields) {
356
		this.browsableFields = browsableFields;
357
	}
358 49890 michele.ar
359 26600 sandro.lab
}