Project

General

Profile

« Previous | Next » 

Revision 54859

[maven-release-plugin] copy for tag dnet-openaire-datasource-manager-1.0.6

View differences:

modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/deploy.info
1
{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-openaire-datasource-manager/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-openaire-datasource-manager"}
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/java/eu/dnetlib/enabling/datasources/DatasourceFunctions.java
1
package eu.dnetlib.enabling.datasources;
2

  
3
import java.io.IOException;
4
import java.sql.Array;
5
import java.sql.Date;
6
import java.sql.SQLException;
7
import java.sql.Timestamp;
8
import java.util.Arrays;
9
import java.util.HashMap;
10
import java.util.List;
11
import java.util.Map;
12
import java.util.Objects;
13
import java.util.stream.Collectors;
14

  
15
import org.apache.commons.lang3.BooleanUtils;
16
import org.apache.commons.lang3.StringUtils;
17
import org.apache.commons.logging.Log;
18
import org.apache.commons.logging.LogFactory;
19
import org.dom4j.Document;
20
import org.dom4j.DocumentException;
21
import org.dom4j.Element;
22
import org.dom4j.io.SAXReader;
23
import org.springframework.core.io.ClassPathResource;
24
import org.springframework.core.io.Resource;
25

  
26
import eu.dnetlib.enabling.datasources.common.Api;
27
import eu.dnetlib.enabling.datasources.common.ApiParam;
28
import eu.dnetlib.enabling.datasources.common.ApiParamImpl;
29
import eu.dnetlib.enabling.datasources.common.Datasource;
30
import eu.dnetlib.enabling.datasources.common.Identity;
31
import eu.dnetlib.enabling.datasources.common.Organization;
32
import eu.dnetlib.enabling.datasources.common.SearchApisEntry;
33
import eu.dnetlib.enabling.datasources.common.SimpleDatasource;
34
import eu.dnetlib.miscutils.datetime.DateUtils;
35

  
36
public class DatasourceFunctions {
37

  
38
	private static final Log log = LogFactory.getLog(DatasourceFunctions.class);
39

  
40
	private static final Resource baseDsProfile = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/datasource_base.xml");
41

  
42
	public static SimpleDatasource mapToSimpleDs(final Map<String, Object> map) {
43
		final SimpleDatasource ds = new SimpleDatasource();
44
		ds.setId(castObject(map.get("id"), String.class));
45
		ds.setName(castObject(map.get("name"), String.class));
46
		ds.setOrigId(castObject(map.get("id"), String.class));
47
		ds.setTypology(castObject(map.get("typology"), String.class));
48
		ds.setValid(true);
49
		try {
50
			final Array arr = castObject(map.get("apis"), Array.class);
51

  
52
			if (arr != null && ((Object[]) arr.getArray()).length > 0) {
53
				ds.setApis(Arrays.stream((Object[]) arr.getArray())
54
						.filter(Objects::nonNull)
55
						.map(Object::toString)
56
						.collect(Collectors.toSet()));
57
			}
58
		} catch (final SQLException e) {
59
			log.error("Error parsing array (apis)", e);
60
			throw new RuntimeException("Error parsing array (apis)", e);
61
		}
62

  
63
		return ds;
64
	}
65

  
66
	public static Datasource<Organization<?>, Identity> mapToDatasource(final Map<String, Object> map) {
67
		final Datasource<Organization<?>, Identity> ds = new Datasource<>();
68
		ds.setId(castObject(map.get("id"), String.class));
69
		ds.setOfficialname(castObject(map.get("officialname"), String.class));
70
		ds.setEnglishname(castObject(map.get("englishname"), String.class));
71
		ds.setWebsiteurl(castObject(map.get("websiteurl"), String.class));
72
		ds.setLogourl(castObject(map.get("logourl"), String.class));
73
		ds.setContactemail(castObject(map.get("contactemail"), String.class));
74
		ds.setLatitude(castObject(map.get("latitude"), Double.class));
75
		ds.setLongitude(castObject(map.get("longitude"), Double.class));
76
		ds.setTimezone(castObject(map.get("timezone"), String.class));
77
		ds.setNamespaceprefix(castObject(map.get("namespaceprefix"), String.class));
78
		ds.setLanguages(castObject(map.get("languages"), String.class));
79
		ds.setOd_contenttypes(castObject(map.get("od_contenttypes"), String.class));
80
		ds.setCollectedfrom(castObject(map.get("collectedfrom"), String.class));
81
		ds.setDateofvalidation(castObject(map.get("dateofvalidation"), Date.class));
82
		ds.setTypology(castObject(map.get("typology"), String.class));
83
		ds.setProvenanceaction(castObject(map.get("provenanceaction"), String.class));
84
		ds.setDateofcollection(castObject(map.get("dateofcollection"), Date.class));
85
		ds.setPlatform(castObject(map.get("platform"), String.class));
86
		ds.setActivationId(castObject(map.get("activationId"), String.class));
87
		ds.setDescription(castObject(map.get("description"), String.class));
88
		ds.setReleasestartdate(castObject(map.get("releasestartdate"), Date.class));
89
		ds.setReleaseenddate(castObject(map.get("releaseenddate"), Date.class));
90
		ds.setMissionstatementurl(castObject(map.get("missionstatementurl"), String.class));
91
		ds.setDataprovider(castObject(map.get("dataprovider"), Boolean.class));
92
		ds.setServiceprovider(castObject(map.get("serviceprovider"), Boolean.class));
93
		ds.setDatabaseaccesstype(castObject(map.get("databaseaccesstype"), String.class));
94
		ds.setDatauploadtype(castObject(map.get("datauploadtype"), String.class));
95
		ds.setDatabaseaccessrestriction(castObject(map.get("databaseaccessrestriction"), String.class));
96
		ds.setDatauploadrestriction(castObject(map.get("datauploadrestriction"), String.class));
97
		ds.setVersioning(castObject(map.get("versioning"), Boolean.class));
98
		ds.setCitationguidelineurl(castObject(map.get("citationguidelineurl"), String.class));
99
		ds.setQualitymanagementkind(castObject(map.get("qualitymanagementkind"), String.class));
100
		ds.setPidsystems(castObject(map.get("pidsystems"), String.class));
101
		ds.setCertificates(castObject(map.get("certificates"), String.class));
102
		ds.setAggregator(castObject(map.get("aggregator"), String.class));
103
		ds.setIssn(castObject(map.get("issn"), String.class));
104
		ds.setEissn(castObject(map.get("eissn"), String.class));
105
		ds.setLissn(castObject(map.get("lissn"), String.class));
106
		ds.setRegisteredby(castObject(map.get("registeredby"), String.class));
107
		ds.setSubjects(castObject(map.get("subjects"), String.class));
108
		ds.setManaged(castObject(map.get("managed"), Boolean.class));
109
		return ds;
110
	}
111

  
112
	public static Identity mapToDsIdentity(final Map<String, Object> map) {
113
		final Identity id = new Identity();
114
		id.setPid(castObject(map.get("pid"), String.class));
115
		id.setIssuertype(castObject(map.get("issuertype"), String.class));
116
		return id;
117
	}
118

  
119
	public static Organization<Datasource<?, ?>> mapToDsOrganization(final Map<String, Object> map) {
120
		final Organization<Datasource<?, ?>> org = new Organization<>();
121
		org.setId(castObject(map.get("id"), String.class));
122
		org.setLegalshortname(castObject(map.get("legalshortname"), String.class));
123
		org.setLegalname(castObject(map.get("legalname"), String.class));
124
		org.setWebsiteurl(castObject(map.get("websiteurl"), String.class));
125
		org.setLogourl(castObject(map.get("logourl"), String.class));
126
		org.setCountry(castObject(map.get("country"), String.class));
127
		org.setCollectedfrom(castObject(map.get("collectedfrom"), String.class));
128
		org.setDateofcollection(castObject(map.get("dateofcollection"), Date.class));
129
		org.setProvenanceaction(castObject(map.get("provenanceaction"), String.class));
130
		return org;
131
	}
132

  
133
	public static SearchApisEntry mapToSearchApisEntry(final Map<String, Object> map) {
134
		final SearchApisEntry a = new SearchApisEntry();
135
		a.setId(castObject(map.get("id"), String.class));
136
		a.setCompliance(castObject(map.get("compliance"), String.class));
137
		a.setActive(castObject(map.get("active"), Boolean.class));
138
		a.setRepoId(castObject(map.get("dsId"), String.class));
139
		a.setRepoName(castObject(map.get("name"), String.class));
140
		a.setRepoCountry(castObject(map.get("country"), String.class));
141
		a.setRepoPrefix(castObject(map.get("prefix"), String.class));
142
		a.setAggrDate(castObject(map.get("aggrDate"), String.class));
143
		a.setAggrTotal(castObject(map.get("aggrTotal"), Integer.class));
144
		a.setProtocol(castObject(map.get("protocol"), String.class));
145
		a.setAlternativeName(castObject(map.get("alternativeName"), String.class));
146
		a.setRepoOrganization(castObject(map.get("organization"), String.class));
147
		return a;
148
	}
149

  
150
	public static Api<ApiParam> mapToApi(final Map<String, Object> map) {
151
		final Api<ApiParam> a = new Api<>();
152
		a.setId(castObject(map.get("id"), String.class));
153
		a.setProtocol(castObject(map.get("protocol"), String.class));
154
		a.setDatasource(castObject(map.get("datasource"), String.class));
155
		a.setContentdescription(castObject(map.get("contentdescription"), String.class));
156
		a.setActive(castObject(map.get("active"), Boolean.class));
157
		a.setRemovable(castObject(map.get("removable"), Boolean.class));
158
		a.setTypology(castObject(map.get("typology"), String.class));
159
		a.setCompatibility(castObject(map.get("compatibility"), String.class));
160
		a.setCompatibilityOverrided(castObject(map.get("isCompatibilityOverrided"), Boolean.class));
161
		a.setMetadataIdentifierPath(castObject(map.get("metadataIdentifierPath"), String.class));
162
		a.setLastCollectionTotal(castObject(map.get("lastCollectionTotal"), Integer.class));
163
		a.setLastCollectionDate(castObject(map.get("lastCollectionDate"), Timestamp.class));
164
		a.setLastCollectionMdid(castObject(map.get("lastCollectionMdid"), String.class));
165
		a.setLastAggregationTotal(castObject(map.get("lastAggregationTotal"), Integer.class));
166
		a.setLastAggregationDate(castObject(map.get("lastAggregationDate"), Timestamp.class));
167
		a.setLastAggregationMdid(castObject(map.get("lastAggregationMdid"), String.class));
168
		a.setLastDownloadTotal(castObject(map.get("lastDownloadTotal"), Integer.class));
169
		a.setLastDownloadDate(castObject(map.get("lastDownloadDate"), Timestamp.class));
170
		a.setLastDownloadObjid(castObject(map.get("lastDownloadObjid"), String.class));
171
		a.setLastValidationJob(castObject(map.get("lastValidationJob"), String.class));
172
		a.setBaseurl(castObject(map.get("baseUrl"), String.class));
173
		try {
174
			final Array arr = castObject(map.get("params"), Array.class);
175
			if (arr != null) {
176
				a.setApiParams(Arrays.stream((Object[]) arr.getArray())
177
						.filter(Objects::nonNull)
178
						.map(Object::toString)
179
						.map(s -> {
180
							final ApiParam p = new ApiParamImpl();
181
							p.setParam(StringUtils.substringBefore(s, "="));
182
							p.setValue(StringUtils.substringAfter(s, "="));
183
							return p;
184
						})
185
						.collect(Collectors.toSet()));
186
			}
187
		} catch (final SQLException e) {
188
			log.error("Error parsing array params", e);
189
			throw new RuntimeException("Error parsing array params", e);
190
		}
191

  
192
		return a;
193
	}
194

  
195
	public static String dsToProfile(final Datasource<Organization<?>, Identity> ds, final List<Api<ApiParam>> apis, final String profId)
196
			throws DocumentException, IOException {
197
		final Document doc = new SAXReader().read(baseDsProfile.getInputStream());
198

  
199
		setValue(doc, "//DATASOURCE_TYPE", ds.getTypology());
200
		setValue(doc, "//DATASOURCE_ORIGINAL_ID", ds.getId());
201
		setValue(doc, "//TYPOLOGY", ds.getPlatform());
202
		setValue(doc, "//OFFICIAL_NAME", ds.getOfficialname());
203
		setValue(doc, "//ENGLISH_NAME", ds.getEnglishname());
204
		setValue(doc, "//ICON_URI", ds.getLogourl());
205
		setValue(doc, "//COUNTRY", ds.getOrganizations().stream().map(Organization::getCountry).findFirst().orElse(""));
206
		setValue(doc, "//LONGITUDE", ds.getLongitude());
207
		setValue(doc, "//LATITUDE", ds.getLatitude());
208
		setValue(doc, "//TIMEZONE", ds.getTimezone());
209
		setValue(doc, "//REPOSITORY_WEBPAGE", ds.getWebsiteurl());
210
		setValue(doc, "//REPOSITORY_INSTITUTION", ds.getOrganizations().stream().map(Organization::getLegalname).findFirst().orElse(""));
211
		setValue(doc, "//ADMIN_INFO", ds.getContactemail());
212
		setValue(doc, "//REGISTERED_BY", ds.getRegisteredby());
213
		setValue(doc, "//LAST_UPDATE/@value", DateUtils.now_ISO8601());
214

  
215
		final Element ef = (Element) doc.selectSingleNode("//EXTRA_FIELDS");
216
		addExtraField(ef, "OpenAireDataSourceId", ds.getId());
217
		addExtraField(ef, "NamespacePrefix", ds.getNamespaceprefix());
218
		addExtraField(ef, "VERIFIED", "NO");
219
		addExtraField(ef, "aggregatorName", ds.getAggregator());
220
		addExtraField(ef, "dateOfValidation", ds.getDateofvalidation());
221
		addExtraField(ef, "dateOfCollection", ds.getDateofcollection());
222
		addExtraField(ef, "mergeHomonyms", "NO");
223
		addExtraField(ef, "ACTID", ds.getActivationId());
224

  
225
		if (apis != null) {
226
			final Element ifaces = (Element) doc.selectSingleNode("//INTERFACES");
227
			apis.forEach(api -> addInterface(ifaces, api));
228
		}
229
		return doc.asXML();
230
	}
231

  
232
	private static void setValue(final Document doc, final String xpath, final Object value) {
233
		if (value != null) {
234
			doc.selectSingleNode(xpath).setText(value.toString());
235
		}
236
	}
237

  
238
	private static void addInterface(final Element ifaces, final Api<?> api) {
239

  
240
		final Element ifc = ifaces.addElement("INTERFACE");
241
		ifc.addAttribute("id", api.getId());
242
		ifc.addAttribute("label", String.format("%s (%s)",
243
				StringUtils.defaultIfBlank(api.getTypology(), "-"),
244
				StringUtils.defaultIfBlank(api.getCompatibility(), "-")));
245
		ifc.addAttribute("typology", StringUtils.defaultIfBlank(api.getTypology(), ""));
246
		ifc.addAttribute("active", "" + BooleanUtils.toBooleanDefaultIfNull(api.getActive(), false));
247
		ifc.addAttribute("compliance", StringUtils.defaultIfBlank(api.getCompatibility(), ""));
248
		ifc.addAttribute("contentDescription", StringUtils.defaultIfBlank(api.getContentdescription(), ""));
249
		ifc.addAttribute("removable", "" + BooleanUtils.toBooleanDefaultIfNull(api.getRemovable(), false));
250

  
251
		final Element ap = ifc.addElement("ACCESS_PROTOCOL");
252
		api.getApiParams().forEach(p -> ap.addAttribute(p.getParam(), p.getValue()));
253
		ap.setText(StringUtils.defaultIfBlank(api.getProtocol(), ""));
254
		ifc.addElement("BASE_URL").setText(StringUtils.defaultIfBlank(api.getBaseurl(), ""));
255

  
256
		if (api.isCompatibilityOverrided()) {
257
			addInterfaceExtraField(ifc, "overriding_compliance", true);
258
		}
259

  
260
		addInterfaceExtraField(ifc, "metadata_identifier_path", api.getMetadataIdentifierPath());
261
		addInterfaceExtraField(ifc, "last_collection_date", api.getLastCollectionDate());
262
		addInterfaceExtraField(ifc, "last_collection_mdId", api.getLastCollectionMdid());
263
		addInterfaceExtraField(ifc, "last_collection_total", api.getLastCollectionTotal());
264
		addInterfaceExtraField(ifc, "last_aggregation_date", api.getLastAggregationDate());
265
		addInterfaceExtraField(ifc, "last_aggregation_mdId", api.getLastAggregationMdid());
266
		addInterfaceExtraField(ifc, "last_aggregation_total", api.getLastAggregationTotal());
267
		addInterfaceExtraField(ifc, "last_download_date", api.getLastDownloadDate());
268
		addInterfaceExtraField(ifc, "last_download_objId", api.getLastDownloadObjid());
269
		addInterfaceExtraField(ifc, "last_download_total", api.getLastDownloadTotal());
270
		addInterfaceExtraField(ifc, "last_validation_job", api.getLastValidationJob());
271
	}
272

  
273
	private static void addExtraField(final Element parent, final String field, final Object value) {
274
		if (value != null && StringUtils.isNotBlank(value.toString())) {
275
			final Element f = parent.addElement("FIELD");
276
			f.addElement("key").setText(field);
277
			f.addElement("value").setText(value.toString());
278
		}
279
	}
280

  
281
	private static void addInterfaceExtraField(final Element parent, final String field, final Object value) {
282
		if (value != null && StringUtils.isNotBlank(value.toString())) {
283
			final Element f = parent.addElement("INTERFACE_EXTRA_FIELD");
284
			f.addAttribute("name", field);
285
			f.setText(value.toString());
286
		}
287
	}
288

  
289
	public static Map<String, Object> dsToMap(final Datasource<Organization<?>, Identity> ds) {
290
		final Map<String, Object> map = new HashMap<>();
291

  
292
		map.put("id", ds.getId());
293
		map.put("officialname", ds.getOfficialname());
294
		map.put("englishname", ds.getEnglishname());
295
		map.put("websiteurl", ds.getWebsiteurl());
296
		map.put("logourl", ds.getLogourl());
297
		map.put("contactemail", ds.getContactemail());
298
		map.put("latitude", ds.getLatitude());
299
		map.put("longitude", ds.getLongitude());
300
		map.put("timezone", ds.getTimezone());
301
		map.put("namespaceprefix", ds.getNamespaceprefix());
302
		map.put("languages", ds.getLanguages());
303
		map.put("od_contenttypes", ds.getOd_contenttypes());
304
		map.put("collectedfrom", ds.getCollectedfrom());
305
		map.put("dateofvalidation", ds.getDateofvalidation());
306
		map.put("typology", ds.getTypology());
307
		map.put("provenanceaction", ds.getProvenanceaction());
308
		map.put("platform", ds.getPlatform());
309
		map.put("activationid", ds.getActivationId());
310
		map.put("description", ds.getDescription());
311
		map.put("releasestartdate", ds.getReleasestartdate());
312
		map.put("releaseenddate", ds.getReleaseenddate());
313
		map.put("missionstatementurl", ds.getMissionstatementurl());
314
		map.put("dataprovider", ds.getDataprovider());
315
		map.put("serviceprovider", ds.getServiceprovider());
316
		map.put("databaseaccesstype", ds.getDatabaseaccesstype());
317
		map.put("datauploadtype", ds.getDatauploadtype());
318
		map.put("databaseaccessrestriction", ds.getDatabaseaccessrestriction());
319
		map.put("datauploadrestriction", ds.getDatauploadrestriction());
320
		map.put("versioning", ds.getVersioning());
321
		map.put("citationguidelineurl", ds.getCitationguidelineurl());
322
		map.put("qualitymanagementkind", ds.getQualitymanagementkind());
323
		map.put("pidsystems", ds.getPidsystems());
324
		map.put("certificates", ds.getCertificates());
325
		map.put("aggregator", ds.getAggregator());
326
		map.put("issn", ds.getIssn());
327
		map.put("eissn", ds.getEissn());
328
		map.put("lissn", ds.getLissn());
329
		map.put("registeredby", ds.getRegisteredby());
330
		map.put("subjects", ds.getSubjects());
331
		map.put("managed", ds.getManaged());
332

  
333
		return map;
334
	}
335

  
336
	public static Map<String, Object> orgToMap(final String dsId, final Organization<?> org) {
337
		final Map<String, Object> map = new HashMap<>();
338
		map.put("dsId", dsId);
339
		map.put("orgId", org.getId());
340
		map.put("legalname", org.getLegalname());
341
		map.put("legalshortname", org.getLegalshortname());
342
		map.put("websiteurl", org.getWebsiteurl());
343
		map.put("logourl", org.getLogourl());
344
		map.put("country", org.getCountry());
345
		return map;
346
	}
347

  
348
	public static Map<String, Object> apiToMap(final Api<?> api) {
349
		final Map<String, Object> map = new HashMap<>();
350
		map.put("apiId", api.getId());
351
		map.put("protocol", api.getProtocol());
352
		map.put("baseUrl", api.getBaseurl());
353
		map.put("dsId", api.getDatasource());
354
		map.put("contentDescription", api.getContentdescription());
355
		map.put("typology", api.getTypology());
356
		map.put("compatibility", api.getCompatibility());
357
		map.put("metadataIdentifierPath", api.getMetadataIdentifierPath());
358
		map.put("removable", api.getRemovable());
359
		// The other fields are not required in the INSERT operation
360
		return map;
361
	}
362

  
363
	@SuppressWarnings("unchecked")
364
	private static <T> T castObject(final Object o, final Class<T> clazz) {
365
		if (o == null) { return null; }
366
		if (clazz.isInstance(o)) { return (T) o; }
367
		throw new IllegalArgumentException("Type not managed: " + clazz.getSimpleName());
368
	}
369

  
370
}
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/java/eu/dnetlib/enabling/datasources/LocalOpenaireDatasourceManager.java
1
package eu.dnetlib.enabling.datasources;
2

  
3
import eu.dnetlib.enabling.datasources.common.Datasource;
4
import eu.dnetlib.enabling.datasources.common.Organization;
5
import eu.dnetlib.enabling.datasources.common.Identity;
6
import eu.dnetlib.enabling.datasources.common.ApiParam;
7
import eu.dnetlib.enabling.datasources.common.Api;
8
import eu.dnetlib.enabling.datasources.common.LocalDatasourceManager;
9

  
10
public interface LocalOpenaireDatasourceManager extends LocalDatasourceManager<Datasource<Organization<?>, Identity>, Api<ApiParam>> {
11

  
12
}
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/java/eu/dnetlib/enabling/datasources/LocalOpenaireDatasourceManagerImpl.java
1
package eu.dnetlib.enabling.datasources;
2

  
3
import java.io.StringWriter;
4
import java.util.*;
5
import java.util.stream.Collectors;
6

  
7
import com.google.common.collect.ImmutableMap;
8
import eu.dnetlib.enabling.datasources.DatasourceManagerClients.AfterSqlUpdate;
9
import eu.dnetlib.enabling.datasources.common.*;
10
import org.apache.commons.io.IOUtils;
11
import org.apache.commons.lang3.StringUtils;
12
import org.apache.commons.lang3.math.NumberUtils;
13
import org.apache.commons.logging.Log;
14
import org.apache.commons.logging.LogFactory;
15
import org.springframework.beans.factory.annotation.Required;
16
import org.springframework.core.io.ClassPathResource;
17
import org.springframework.core.io.Resource;
18
import org.springframework.transaction.annotation.Transactional;
19

  
20
public class LocalOpenaireDatasourceManagerImpl implements LocalOpenaireDatasourceManager {
21

  
22
	private DatasourceManagerClients datasourceManagerClients;
23

  
24
	private List<DbBrowsableField> browsableFields;
25

  
26
	public static final String QUERY_BASEDIR = "/eu/dnetlib/enabling/datasources/queries/";
27

  
28
	private static final Resource searchDsByType = new ClassPathResource(QUERY_BASEDIR + "searchDsByType.sql");
29
	private static final Resource searchApis = new ClassPathResource(QUERY_BASEDIR + "searchApisNormal.sql");
30
	private static final Resource searchApisUsingField = new ClassPathResource(QUERY_BASEDIR + "searchApisUsingField.sql");
31
	private static final Resource addDs = new ClassPathResource(QUERY_BASEDIR + "addDatasource.sql");
32
	private static final Resource addOrg = new ClassPathResource(QUERY_BASEDIR + "addOrganization.sql");
33
	private static final Resource deleteDs = new ClassPathResource(QUERY_BASEDIR + "deleteDatasource.sql");
34
	private static final Resource setActive = new ClassPathResource(QUERY_BASEDIR + "setActive.sql");
35
	private static final Resource setManaged = new ClassPathResource(QUERY_BASEDIR + "setManaged.sql");
36
	private static final Resource setCompliance = new ClassPathResource(QUERY_BASEDIR + "setCompliance.sql");
37
	private static final Resource overrideCompliance = new ClassPathResource(QUERY_BASEDIR + "overrideCompliance.sql");
38
	private static final Resource resetCompliance = new ClassPathResource(QUERY_BASEDIR + "resetCompliance.sql");
39
	private static final Resource setLastCollectionInfo = new ClassPathResource(QUERY_BASEDIR + "setLastCollectionInfo.sql");
40
	private static final Resource setLastAggregationInfo = new ClassPathResource(QUERY_BASEDIR + "setLastAggregationInfo.sql");
41
	private static final Resource setLastDownloadInfo = new ClassPathResource(QUERY_BASEDIR + "setLastDownloadInfo.sql");
42
	private static final Resource setLastValidationJob = new ClassPathResource(QUERY_BASEDIR + "setLastValidationJob.sql");
43
	private static final Resource resetLastOperationsInfo = new ClassPathResource(QUERY_BASEDIR + "resetLastOperationsInfo.sql");
44
	private static final Resource insertApiParam = new ClassPathResource(QUERY_BASEDIR + "insertApiParam.sql");
45
	private static final Resource insertApi = new ClassPathResource(QUERY_BASEDIR + "insertApi.sql");
46

  
47
	private static final Log log = LogFactory.getLog(LocalOpenaireDatasourceManagerImpl.class);
48

  
49
	private static final String REPO_PROFILEID_SUFFIX = "_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=";
50

  
51
	@Override
52
	@Transactional(readOnly = true)
53
	public Set<String> listManagedDatasourceIds() throws DsmRuntimeException {
54
		try {
55
			return datasourceManagerClients.searchSQL("SELECT id FROM dsm_datasources WHERE managed = true", new HashMap<>())
56
					.stream()
57
					.map(m -> (String) m.get("id"))
58
					.collect(Collectors.toCollection(HashSet::new));
59
		} catch (final DsmException e) {
60
			throw new DsmRuntimeException(e);
61
		}
62
	}
63

  
64
	@Override
65
	@Transactional(readOnly = true)
66
	public List<SimpleDatasource> searchDatasourcesByType(final String type) throws DsmException {
67

  
68
		return datasourceManagerClients.searchSQL(searchDsByType, ImmutableMap.of("type", type))
69
				.stream()
70
				.map(DatasourceFunctions::mapToSimpleDs)
71
				.collect(Collectors.toList());
72
	}
73

  
74
	@Override
75
	@Transactional(readOnly = true)
76
	public List<? extends SearchApisEntry> searchApis(final String field, final Object value) throws DsmException {
77
		try {
78
			final StringWriter sql = new StringWriter();
79

  
80
			if (field.equalsIgnoreCase("__search__")) {
81
				sql.append(IOUtils.toString(searchApis.getInputStream()));
82
			} else {
83
				sql.append(IOUtils.toString(searchApisUsingField.getInputStream()));
84
				sql.append(field);
85
				sql.append("::text = ");
86
				sql.append(":value");
87
			}
88

  
89
			return datasourceManagerClients
90
					.searchSQL(sql.toString(), ImmutableMap.of("value", field.equalsIgnoreCase("__search__") ? "%" + value + "%" : value))
91
					.stream()
92
					.map(DatasourceFunctions::mapToSearchApisEntry)
93
					.collect(Collectors.toList());
94

  
95
		} catch (final Exception e) {
96
			log.error("Error searching field " + field + " - value: " + value, e);
97
		}
98
		return new ArrayList<>();
99
	}
100

  
101
	@Override
102
	@Transactional
103
	public void saveDs(final Datasource<Organization<?>, Identity> ds) throws DsmException {
104

  
105
		if (StringUtils.isBlank(ds.getAggregator())) {
106
			ds.setAggregator("OPENAIRE");
107
		}
108

  
109
		ds.setManaged(true);
110

  
111
		datasourceManagerClients.updateSQL(ds.getId(), addDs, AfterSqlUpdate.NONE, DatasourceFunctions.dsToMap(ds));
112

  
113
		if (ds.getOrganizations() != null) {
114
			for (final Organization<?> org : ds.getOrganizations()) {
115
				final Map<String, Object> orgParams = DatasourceFunctions.orgToMap(ds.getId(), org);
116
				datasourceManagerClients.updateSQL(ds.getId(), addOrg, AfterSqlUpdate.NONE, orgParams);
117
			}
118
		}
119

  
120
		datasourceManagerClients.regenerateProfile(ds.getId());
121
	}
122

  
123
	@Override
124
	@Transactional
125
	public void deleteDs(final String dsId) throws DsmException {
126
		datasourceManagerClients.updateSQL(fixDsId(dsId), deleteDs, AfterSqlUpdate.DELETE_DS_PROFILE, ImmutableMap.of("dsId", dsId));
127
	}
128

  
129
	@Override
130
	public Datasource<Organization<?>, Identity> getDs(final String dsId) throws DsmException {
131
		return datasourceManagerClients.getDatasourceById(fixDsId(dsId));
132
	}
133

  
134
	@Override
135
	public void regenerateProfiles() throws DsmException {
136
		datasourceManagerClients.regenerateProfiles();
137
	}
138

  
139
	@Override
140
	public List<Api<ApiParam>> getApis(final String dsId) throws DsmException {
141
		return datasourceManagerClients.getApis(fixDsId(dsId));
142
	}
143

  
144
	@Override
145
	public void setManaged(final String dsId, final boolean managed) throws DsmException {
146
		final String id = fixDsId(dsId);
147

  
148
		final Map<String, Object> params = new HashMap<>();
149
		params.put("managed", managed);
150
		params.put("dsId", id);
151

  
152
		datasourceManagerClients.updateSQL(id, setManaged, AfterSqlUpdate.UPDATE_DS_PROFILE, params);
153

  
154
	}
155

  
156
	@Override
157
	public boolean isManaged(final String dsId) throws DsmException {
158
		final String q = "SELECT * from dsm_datasources WHERE id = :dsId AND managed = true";
159
		return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", fixDsId(dsId))).isEmpty();
160
	}
161

  
162
	@Override
163
	@Transactional
164
	public void setActive(final String dsId, final String apiId, final boolean active) throws DsmException {
165
		final String id = fixDsId(dsId);
166

  
167
		final Map<String, Object> params = new HashMap<>();
168
		params.put("active", active);
169
		params.put("apiId", apiId);
170
		params.put("dsId", id);
171

  
172
		datasourceManagerClients.updateSQL(id, setActive, AfterSqlUpdate.NONE, params);
173
		if (!active) {
174
			datasourceManagerClients.updateSQL(dsId, resetLastOperationsInfo, AfterSqlUpdate.NONE, params);
175
		}
176

  
177
		setManaged(id, true); // It also update the IS profile
178
	}
179

  
180
	@Override
181
	@Transactional(readOnly = true)
182
	public boolean isActive(final String dsId, final String apiId) throws DsmException {
183
		final String q = "SELECT * from dsm_api WHERE id = :apiId AND datasource = :dsId AND active = true";
184
		return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", fixDsId(dsId), "apiId", apiId)).isEmpty();
185
	}
186

  
187
	@Override
188
	@Transactional(readOnly = true)
189
	public boolean isRemovable(final String dsId, final String apiId) throws DsmException {
190
		final String q = "SELECT * from dsm_api WHERE id = :apiId AND datasource = :dsId AND active != true AND removable = true";
191
		return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", fixDsId(dsId), "apiId", apiId)).isEmpty();
192
	}
193

  
194
	@Override
195
	@Transactional
196
	public void updateCompliance(final String dsId, final String apiId, final String level, final boolean override) throws DsmException {
197
		final String id = fixDsId(dsId);
198

  
199
		if (!override) {
200
			datasourceManagerClients.updateSQL(id, setCompliance, AfterSqlUpdate.NONE, ImmutableMap.of("level", level, "apiId", apiId, "dsId", id));
201
		} else if (level != null) {
202
			datasourceManagerClients.updateSQL(id, overrideCompliance, AfterSqlUpdate.NONE, ImmutableMap.of("level", level, "apiId", apiId, "dsId", id));
203
		} else {
204
			datasourceManagerClients.updateSQL(id, resetCompliance, AfterSqlUpdate.NONE, ImmutableMap.of("apiId", apiId, "dsId", id));
205
		}
206

  
207
		setManaged(id, true); // It also update the IS profile
208
	}
209

  
210
	@Override
211

  
212
	public void setLastCollectionInfo(final String dsId, final String apiId, final String mdId, final Integer size, final Date date)
213
			throws DsmException {
214
		setLastOperationInfo(setLastCollectionInfo, fixDsId(dsId), apiId, mdId, size, date);
215
	}
216

  
217
	@Override
218
	public void setLastAggregationInfo(final String dsId, final String apiId, final String mdId, final Integer size, final Date date)
219
			throws DsmException {
220
		setLastOperationInfo(setLastAggregationInfo, fixDsId(dsId), apiId, mdId, size, date);
221
	}
222

  
223
	@Override
224
	public void setLastDownloadInfo(final String dsId, final String apiId, final String objId, final Integer size, final Date date)
225
			throws DsmException {
226
		setLastOperationInfo(setLastDownloadInfo, fixDsId(dsId), apiId, objId, size, date);
227
	}
228

  
229
	@Override
230
	public void setLastValidationJob(String dsId, String apiId, String jobId) throws DsmException {
231
		final Map<String, Object> params = new HashMap<>();
232
		params.put("dsId", dsId);
233
		params.put("apiId", apiId);
234
		params.put("jobId", jobId);
235

  
236
		datasourceManagerClients.updateSQL(dsId, setLastValidationJob, AfterSqlUpdate.NONE, params);
237
		setManaged(dsId, true); // It also update the IS profile
238
	}
239

  
240
	@Transactional
241
	protected void setLastOperationInfo(final Resource sqlResource, final String dsId, final String apiId, final String mdId, final Integer size, final Date date)
242
			throws DsmException {
243
		final Map<String, Object> params = new HashMap<>();
244
		params.put("dsId", dsId);
245
		params.put("apiId", apiId);
246
		params.put("mdId", mdId);
247
		params.put("total", size);
248
		if (date != null) {
249
			params.put("date", new java.sql.Timestamp(date.getTime()));
250
		}
251

  
252
		datasourceManagerClients.updateSQL(dsId, sqlResource, AfterSqlUpdate.NONE, params);
253

  
254
		setManaged(dsId, true); // It also update the IS profile
255
	}
256

  
257
	@Override
258
	@Transactional
259
	public void updateApiDetails(final String dsId,
260
			final String apiId,
261
			final String metadataIdentifierPath,
262
			final String baseUrl,
263
			final Map<String, String> params)
264
			throws DsmException {
265

  
266
		final String id = fixDsId(dsId);
267

  
268
		// Delete old params
269
		datasourceManagerClients.updateSQL(id, "DELETE FROM dsm_apiparams WHERE api = :api", AfterSqlUpdate.NONE, ImmutableMap.of("api", apiId));
270

  
271
		// Insert new params
272
		for (final Map.Entry<String, String> e : params.entrySet()) {
273
			final Map<String, Object> sqlParams = ImmutableMap.of("param", e.getKey(), "value", e.getValue(), "api", apiId);
274
			datasourceManagerClients.updateSQL(id, insertApiParam, AfterSqlUpdate.NONE, sqlParams);
275
		}
276

  
277
		// Update the BaseURL
278
		datasourceManagerClients.updateSQL(id,
279
				"UPDATE dsm_api SET baseurl = :baseurl WHERE id = :api",
280
				AfterSqlUpdate.NONE,
281
				ImmutableMap.of("baseurl", baseUrl, "api", apiId));
282

  
283
		// Update the metadata_identifier_path
284
		datasourceManagerClients.updateSQL(id,
285
				"UPDATE dsm_api SET metadata_identifier_path = :path WHERE id = :api",
286
				AfterSqlUpdate.NONE,
287
				ImmutableMap.of("path", metadataIdentifierPath, "api", apiId));
288

  
289
		setManaged(id, true); // It also update the IS profile
290
	}
291

  
292
	@Override
293
	public List<? extends BrowsableField> listBrowsableFields() throws DsmException {
294
		return getBrowsableFields();
295
	}
296

  
297
	@Override
298
	public List<BrowseTerm> browseField(final String field) throws DsmException {
299
		final Optional<DbBrowsableField> bf = getBrowsableFields()
300
				.stream()
301
				.filter(f -> f.getId().equals(field))
302
				.findFirst();
303

  
304
		if (bf.isPresent()) {
305
			return datasourceManagerClients.searchSQL(bf.get().getSql(), new HashMap<>())
306
					.stream()
307
					.filter(m -> m.get("term") != null)
308
					.filter(m -> m.get("count") != null)
309
					.filter(m -> StringUtils.isNotBlank(m.get("term").toString()))
310
					.map(m -> new BrowseTermImpl(m.get("term").toString(), NumberUtils.toInt(m.get("count").toString(), 0)))
311
					.collect(Collectors.toList());
312
		} else {
313
			log.error("Not browsable field:" + field);
314
			throw new DsmException("Not browsable field:" + field);
315
		}
316
	}
317

  
318
	@Override
319
	@Transactional
320
	public void addApi(final Api<ApiParam> api) throws DsmException {
321

  
322
		datasourceManagerClients.updateSQL(api.getDatasource(), insertApi, AfterSqlUpdate.NONE, DatasourceFunctions.apiToMap(api));
323

  
324
		if (api.getApiParams() != null) {
325
			api.getApiParams().forEach(p -> {
326
				final ImmutableMap<String, Object> sqlParams = ImmutableMap.of("param", p.getParam(), "value", p.getValue(), "api", api.getId());
327
				try {
328
					datasourceManagerClients.updateSQL(api.getDatasource(), insertApiParam, AfterSqlUpdate.NONE, sqlParams);
329
				} catch (final DsmException e) {
330
					throw new RuntimeException(e);
331
				}
332
			});
333
		}
334

  
335
		setManaged(api.getDatasource(), true); // It also update the IS profile
336
	}
337

  
338
	@Override
339
	@Transactional
340
	public void deleteApi(final String dsId, final String apiId) throws DsmException {
341
		if (!isRemovable(dsId, apiId)) { throw new DsmException("The api " + apiId + " can't be deleted"); }
342

  
343
		datasourceManagerClients.updateSQL(dsId, "DELETE FROM dsm_apiparams WHERE api = :api", AfterSqlUpdate.NONE, ImmutableMap.of("api", apiId));
344
		datasourceManagerClients.updateSQL(dsId, "DELETE FROM dsm_api WHERE id = :api", AfterSqlUpdate.NONE, ImmutableMap.of("api", apiId));
345

  
346
		setManaged(dsId, true); // It also update the IS profile
347
	}
348

  
349
	public DatasourceManagerClients getDatasourceManagerClients() {
350
		return datasourceManagerClients;
351
	}
352

  
353
	private String fixDsId(final String dsId) throws DsmException {
354
		return dsId.endsWith(REPO_PROFILEID_SUFFIX) ? datasourceManagerClients.findDatasourceId(dsId) : dsId;
355
	}
356

  
357
	@Required
358
	public void setDatasourceManagerClients(final DatasourceManagerClients datasourceManagerClients) {
359
		this.datasourceManagerClients = datasourceManagerClients;
360
	}
361

  
362
	public List<DbBrowsableField> getBrowsableFields() {
363
		return browsableFields;
364
	}
365

  
366
	@Required
367
	public void setBrowsableFields(final List<DbBrowsableField> browsableFields) {
368
		this.browsableFields = browsableFields;
369
	}
370

  
371
}
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/java/eu/dnetlib/enabling/datasources/DatasourceManagerClients.java
1
package eu.dnetlib.enabling.datasources;
2

  
3
import java.io.StringReader;
4
import java.text.ParseException;
5
import java.util.Date;
6
import java.util.HashMap;
7
import java.util.List;
8
import java.util.Map;
9
import java.util.stream.Collectors;
10

  
11
import org.apache.commons.io.IOUtils;
12
import org.apache.commons.logging.Log;
13
import org.apache.commons.logging.LogFactory;
14
import org.dom4j.Document;
15
import org.dom4j.io.SAXReader;
16
import org.quartz.CronExpression;
17
import org.springframework.beans.factory.annotation.Autowired;
18
import org.springframework.beans.factory.annotation.Required;
19
import org.springframework.core.io.ClassPathResource;
20
import org.springframework.core.io.Resource;
21
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
22

  
23
import com.google.common.collect.ImmutableMap;
24

  
25
import eu.dnetlib.enabling.datasources.common.Api;
26
import eu.dnetlib.enabling.datasources.common.ApiParam;
27
import eu.dnetlib.enabling.datasources.common.Datasource;
28
import eu.dnetlib.enabling.datasources.common.DsmException;
29
import eu.dnetlib.enabling.datasources.common.Identity;
30
import eu.dnetlib.enabling.datasources.common.Organization;
31
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpDocumentNotFoundException;
32
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
33
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
34
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
35
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
36
import org.springframework.transaction.annotation.Transactional;
37

  
38
public class DatasourceManagerClients {
39

  
40
	private static final String REPOSITORY_SERVICE_RESOURCE_TYPE = "RepositoryServiceResourceType";
41
	private static final Log log = LogFactory.getLog(DatasourceManagerClients.class);
42

  
43
	private static final Resource dsQuery = new ClassPathResource(LocalOpenaireDatasourceManagerImpl.QUERY_BASEDIR + "getDatasource.sql");
44
	private static final Resource dsIdentitiesQuery = new ClassPathResource(LocalOpenaireDatasourceManagerImpl.QUERY_BASEDIR + "dsIdentitiesQuery.sql");
45
	private static final Resource dsOrganizationsQuery = new ClassPathResource(LocalOpenaireDatasourceManagerImpl.QUERY_BASEDIR + "dsOrganizationsQuery.sql");
46
	private static final Resource listApisByDsId = new ClassPathResource(LocalOpenaireDatasourceManagerImpl.QUERY_BASEDIR + "listApisByDsId.sql");
47
	private static final Resource isDefinedParamQuery = new ClassPathResource(LocalOpenaireDatasourceManagerImpl.QUERY_BASEDIR + "isDefinedParam.sql");
48

  
49
	private NamedParameterJdbcTemplate jdbcTemplate;
50

  
51
	public enum AfterSqlUpdate {
52
		DELETE_DS_PROFILE, UPDATE_DS_PROFILE, NONE
53
	}
54

  
55
	@Autowired
56
	private UniqueServiceLocator serviceLocator;
57

  
58
	public String findDatasourceId(final String profileId) throws DsmException {
59
		try {
60
			return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(
61
					"/*[.//RESOURCE_IDENTIFIER/@value='" + profileId + "']//EXTRA_FIELDS/FIELD[./key='OpenAireDataSourceId']/value/text()");
62
		} catch (final Exception e) {
63
			log.error("Error finding dsId of profile " + profileId, e);
64
			throw new DsmException(-1, "Error finding dsId of profile " + profileId, e);
65
		}
66
	}
67

  
68
	public String getDatasourceProfile(final String dsId) throws DsmException {
69
		try {
70
			return serviceLocator.getService(ISLookUpService.class)
71
					.getResourceProfileByQuery(
72
							"collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')/*[.//EXTRA_FIELDS/FIELD[./key='OpenAireDataSourceId']/value/text() = '"
73
									+ dsId + "']");
74
		} catch (final Exception e) {
75
			return null;
76
		}
77
	}
78

  
79
	public boolean deleteProfile(final String dsId) throws DsmException {
80
		try {
81
			final SAXReader reader = new SAXReader();
82

  
83
			final String profile = getDatasourceProfile(dsId);
84

  
85
			if (profile != null) {
86
				final Document docOld = reader.read(new StringReader(profile));
87
				final String profId = docOld.valueOf("//RESOURCE_IDENTIFIER/@value");
88
				serviceLocator.getService(ISRegistryService.class).deleteProfile(profId);
89
			}
90
			return true;
91
		} catch (final Exception e) {
92
			log.error("Error deleting profile", e);
93
			throw new DsmException(-1, "Error deleting profile", e);
94
		}
95
	}
96

  
97
	public boolean regenerateProfile(final String dsId) throws DsmException {
98
		return regenerateProfile(dsId, serviceLocator.getService(ISRegistryService.class));
99
	}
100

  
101
	public boolean regenerateProfile(final String dsId, final ISRegistryService registry) throws DsmException {
102

  
103
		final Datasource<Organization<?>, Identity> ds = getDatasourceById(dsId);
104
		final List<Api<ApiParam>> apis = getApis(dsId);
105

  
106
		try {
107

  
108
			final String oldProfile = getDatasourceProfile(dsId);
109

  
110
			if (oldProfile != null) {
111
				final Document docOld = new SAXReader().read(new StringReader(oldProfile));
112
				final String profId = docOld.valueOf("//RESOURCE_IDENTIFIER/@value");
113
				final String profile = DatasourceFunctions.dsToProfile(ds, apis, profId);
114
				registry.updateProfile(profId, profile, REPOSITORY_SERVICE_RESOURCE_TYPE);
115
				log.info("Profile " + profId + " UPDATED for ds " + dsId);
116
			} else {
117
				final String profile = DatasourceFunctions.dsToProfile(ds, apis, "");
118
				final String profId = registry.registerProfile(profile);
119
				log.info("Valid Profile " + profId + " REGISTERED for ds " + dsId);
120
			}
121
			return true;
122
		} catch (final Exception e) {
123
			log.error("Error saving profile, id: " + dsId, e);
124
			throw new DsmException(-1, "Error regenerating profile", e);
125
		}
126
	}
127

  
128

  
129
	@Transactional(readOnly = true)
130
	public List<Map<String, Object>> searchSQL(final String sql, final Map<String, Object> sqlParams) throws DsmException {
131
		try {
132
			log.debug("Executing SQL: " + sql);
133
			return jdbcTemplate.queryForList(sql, sqlParams);
134
		} catch (final Exception e) {
135
			log.error("Error executing sql", e);
136

  
137
			throw new DsmException(-1, "Error obtaining datasources from db", e);
138
		}
139
	}
140

  
141

  
142
	@Transactional(readOnly = true)
143
	public List<Map<String, Object>> searchSQL(final Resource sqlResource, final Map<String, Object> sqlParams) throws DsmException {
144
		try {
145
			return searchSQL(IOUtils.toString(sqlResource.getInputStream()), sqlParams);
146
		} catch (final Exception e) {
147
			log.error("Error executing sql", e);
148
			throw new DsmException(-1, "Error obtaining datasources from db", e);
149
		}
150
	}
151

  
152

  
153
	@Transactional
154
	public void updateSQL(final String dsId, final String sql, final AfterSqlUpdate op, final Map<String, Object> sqlparams)
155
			throws DsmException {
156
		log.debug("Executing query SQL: " + sql);
157

  
158
		jdbcTemplate.update(sql, sqlparams);
159

  
160
		switch (op) {
161
		case DELETE_DS_PROFILE:
162
			deleteProfile(dsId);
163
			break;
164
		case UPDATE_DS_PROFILE:
165

  
166
			regenerateProfile(dsId, serviceLocator.getService(ISRegistryService.class));
167
			break;
168
		default:
169
			break;
170
		}
171

  
172
	}
173
	@Transactional
174
	public void updateSQL(final String dsId, final Resource sqlResource, final AfterSqlUpdate op, final Map<String, Object> sqlparams)
175
			throws DsmException {
176
		try {
177
			updateSQL(dsId, IOUtils.toString(sqlResource.getInputStream()), op, sqlparams);
178
		} catch (final Exception e) {
179
			log.error("Error in updateSQL", e);
180
			throw new DsmException(-1, "Error in updateSQL", e);
181
		}
182
	}
183

  
184
	@Transactional(readOnly = true)
185
	public Datasource<Organization<?>, Identity> getDatasourceById(final String id) throws DsmException {
186
		final List<Map<String, Object>> list = searchSQL(dsQuery, ImmutableMap.of("dsId", id));
187

  
188
		if (list.size() != 1) { throw new DsmException("Invalid number of ds with id: " + id); }
189

  
190
		final Datasource<Organization<?>, Identity> ds = DatasourceFunctions.mapToDatasource(list.get(0));
191
		ds.setIdentities(searchSQL(dsIdentitiesQuery, ImmutableMap.of("dsId", id))
192
				.stream()
193
				.map(DatasourceFunctions::mapToDsIdentity)
194
				.collect(Collectors.toSet()));
195
		ds.setOrganizations(searchSQL(dsOrganizationsQuery, ImmutableMap.of("dsId", id))
196
				.stream()
197
				.map(DatasourceFunctions::mapToDsOrganization)
198
				.collect(Collectors.toSet()));
199

  
200
		return ds;
201
	}
202

  
203

  
204
	@Transactional(readOnly = true)
205
	public List<Api<ApiParam>> getApis(final String dsId) throws DsmException {
206

  
207
		return searchSQL(listApisByDsId, ImmutableMap.of("dsId", dsId))
208
				.stream()
209
				.map(DatasourceFunctions::mapToApi)
210
				.collect(Collectors.toList());
211
	}
212

  
213
	@Transactional(readOnly = true)
214
	public boolean isDefinedParam(final String apiId, final String param) throws DsmException {
215
		return !searchSQL(isDefinedParamQuery, ImmutableMap.of("apiId", apiId, "param", param)).isEmpty();
216
	}
217

  
218
	public Date findNextScheduledExecution(final String dsId, final String ifaceId) throws DsmException {
219
		final String xquery = "/*[.//DATAPROVIDER/@interface='" + ifaceId + "' and .//SCHEDULING/@enabled='true']//CRON/text()";
220
		try {
221
			final String cronExpression = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xquery);
222
			final CronExpression cron = new CronExpression(cronExpression);
223
			return cron.getNextValidTimeAfter(new Date());
224
		} catch (final ISLookUpDocumentNotFoundException e) {
225
			// When the value is not found a null value must be returned
226
			return null;
227
		} catch (final ISLookUpException e) {
228
			log.error("Error in xquery: " + xquery, e);
229
			throw new DsmException(-1, "Error in xquery: " + xquery, e);
230
		} catch (final ParseException e) {
231
			log.error("Error parsing cron expression", e);
232
			throw new DsmException(-1, "Error parsing cron expression", e);
233
		}
234
	}
235

  
236
	@Transactional(readOnly = true)
237
	public void regenerateProfiles() throws DsmException {
238
		final ISRegistryService registry = serviceLocator.getService(ISRegistryService.class);
239
		searchSQL("SELECT id FROM dsm_datasources", new HashMap<>()).stream()
240
				.map(m -> m.get("id").toString())
241
				.forEach(id -> {
242
					try {
243
						regenerateProfile(id, registry);
244
					} catch (final DsmException e) {
245
						log.error("Error regeneating profile: " + id, e);
246
					}
247
				});
248
	}
249

  
250
	public NamedParameterJdbcTemplate getJdbcTemplate() {
251
		return jdbcTemplate;
252
	}
253

  
254
	@Required
255
	public void setJdbcTemplate(final NamedParameterJdbcTemplate jdbcTemplate) {
256
		this.jdbcTemplate = jdbcTemplate;
257
	}
258

  
259
}
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/java/eu/dnetlib/enabling/datasources/DbBrowsableField.java
1
package eu.dnetlib.enabling.datasources;
2

  
3
import eu.dnetlib.enabling.datasources.common.BrowsableField;
4

  
5
public class DbBrowsableField extends BrowsableField {
6

  
7
	private String sql;
8

  
9
	public DbBrowsableField() {
10
		super();
11
	}
12

  
13
	public DbBrowsableField(final String id, final String label, final String sql) {
14
		super(id, label);
15
		this.sql = sql;
16
	}
17

  
18
	public String getSql() {
19
		return sql;
20
	}
21

  
22
	public void setSql(String sql) {
23
		this.sql = sql;
24
	}
25

  
26
}
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/resources/db_fixes/script.sql
1
alter table dsm_api add column compatibility_override varchar(255) default NULL;
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/resources/eu/dnetlib/enabling/datasources/applicationContext-dnet-openaire-datasource-manager.xml
1
<?xml version="1.0" encoding="UTF-8"?>
2
<beans xmlns="http://www.springframework.org/schema/beans"
3
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws"
4
	xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing"
5
	xmlns:p="http://www.springframework.org/schema/p" xmlns:http="http://cxf.apache.org/transports/http/configuration"
6
	xmlns:t="http://dnetlib.eu/springbeans/t" xmlns:template="http://dnetlib.eu/springbeans/template"
7
	xmlns:util="http://www.springframework.org/schema/util"
8
	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
9
                                    http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd
10
                                    http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd
11
                                    http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd
12
                            http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd
13
                            http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.0.xsd
14
                            http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">
15

  
16
	<bean id="localOpenaireDatasourceManager"
17
		class="eu.dnetlib.enabling.datasources.LocalOpenaireDatasourceManagerImpl"
18
		p:datasourceManagerClients-ref="datasourceManagerClients">
19
		<property name="browsableFields">
20
			<list>
21
				<bean class="eu.dnetlib.enabling.datasources.DbBrowsableField"
22
					p:id="country" p:label="Datasource countries"
23
					p:sql="select o.country as term, count(*) as count from dsm_api a left outer join dsm_datasources d on (a.datasource = d.id) left outer join dsm_datasource_organization dao on (d.id = dao.datasource) left outer join dsm_organizations o on (dao.organization = o.id) group by o.country order by count desc" />
24
				<bean class="eu.dnetlib.enabling.datasources.DbBrowsableField"
25
					p:id="type" p:label="API typologies"
26
					p:sql="select a.typology as term, count(*) as count from dsm_api a group by a.typology order by count desc" />
27
				<bean class="eu.dnetlib.enabling.datasources.DbBrowsableField"
28
					p:id="protocol" p:label="API protocols"
29
					p:sql="select a.protocol as term, count(*) as count from dsm_api a group by a.protocol order by count desc" />
30
				<bean class="eu.dnetlib.enabling.datasources.DbBrowsableField"
31
					p:id="compliance" p:label="API compatibility levels"
32
					p:sql="select coalesce(a.compatibility_override, a.compatibility) as term, count(*) as count from dsm_api a group by coalesce(a.compatibility_override, a.compatibility) order by count desc" />
33
				<bean class="eu.dnetlib.enabling.datasources.DbBrowsableField"
34
					p:id="active" p:label="API activation"
35
					p:sql="select a.active as term, count(*) as count from dsm_api a group by a.active order by count desc" />
36
			</list>
37
		</property>
38
	</bean>
39

  
40
	<bean id="datasourceManagerClients"
41
		class="eu.dnetlib.enabling.datasources.DatasourceManagerClients">
42
		<property name="jdbcTemplate">
43
			<bean class="org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate">
44
				<constructor-arg>
45
					<bean class="org.springframework.jdbc.datasource.DriverManagerDataSource"
46
						p:driverClassName="org.postgresql.Driver" p:url="${dnet.openaire.db.url}"
47
						p:username="${dnet.openaire.db.username}" p:password="${dnet.openaire.db.password}" />
48
				</constructor-arg>
49
			</bean>
50
		</property>
51
	</bean>
52

  
53
</beans>
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/resources/eu/dnetlib/enabling/datasources/applicationContext-dnet-openaire-datasource-manager.properties
1
dnet.openaire.db.url      = jdbc:postgresql://localhost:5432/dnet_openaire
2
dnet.openaire.db.username = dnet
3
dnet.openaire.db.password =
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/resources/eu/dnetlib/enabling/datasources/queries/setLastDownloadInfo.sql
1
UPDATE dsm_api
2
SET (last_download_objid, last_download_total, last_download_date) = (:mdId, :total, :date)
3
WHERE id = :apiId AND datasource = :dsId
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/resources/eu/dnetlib/enabling/datasources/queries/setLastValidationJob.sql
1
UPDATE dsm_api
2
SET (last_validation_job) = (:jobId)
3
WHERE id = :apiId AND datasource = :dsId
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/resources/eu/dnetlib/enabling/datasources/queries/resetLastOperationsInfo.sql
1
UPDATE dsm_api
2
SET 	(last_aggregation_mdid, last_aggregation_total, last_aggregation_date, last_collection_mdid, last_collection_total, last_collection_date, last_download_objid, last_download_total, last_download_date) = (null, null, null, null, null, null, null, null, null)
3
WHERE id = :apiId AND datasource = :dsId
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/resources/eu/dnetlib/enabling/datasources/queries/setCompliance.sql
1
UPDATE dsm_api SET compatibility = :level WHERE id = :apiId AND datasource = :dsId
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/resources/eu/dnetlib/enabling/datasources/queries/listApisByDsId.sql
1
SELECT
2
	a.id                                                AS "id",
3
	a.protocol                                          AS "protocol",
4
	a.baseurl                                           AS "baseUrl",
5
	a.datasource                                        AS "datasource",
6
	a.contentdescription                                AS "contentdescription",
7
	a.active                                            AS "active",
8
	a.removable                                         AS "removable",
9
	a.typology                                          AS "typology",
10
	coalesce(a.compatibility_override, a.compatibility) AS "compatibility",
11
	a.compatibility_override IS NOT null                AS "isCompatibilityOverrided",
12
	a.metadata_identifier_path                          AS "metadataIdentifierPath",
13
	a.last_collection_total                             AS "lastCollectionTotal",
14
	a.last_collection_date                              AS "lastCollectionDate",
15
	a.last_collection_mdid                              AS "lastCollectionMdid",
16
	a.last_aggregation_total                            AS "lastAggregationTotal",
17
	a.last_aggregation_date                             AS "lastAggregationDate",
18
	a.last_aggregation_mdid                             AS "lastAggregationMdid",
19
	a.last_download_total                               AS "lastDownloadTotal",
20
	a.last_download_date                                AS "lastDownloadDate",
21
	a.last_download_objid                               AS "lastDownloadObjid",
22
	a.last_validation_job                               AS "lastValidationJob",
23
	array_agg(p.param || '=' || p.value)                AS "params"
24
FROM 
25
	dsm_api a
26
	LEFT OUTER JOIN dsm_apiparams p on (a.id = p.api)
27
WHERE
28
	a.datasource = :dsId
29
GROUP BY
30
	a.id,
31
	a.protocol,
32
	a.baseurl,
33
	a.datasource,
34
	a.contentdescription,
35
	a.active,
36
	a.removable,
37
	a.typology,
38
	a.compatibility_override,
39
	a.compatibility,
40
	a.metadata_identifier_path,
41
	a.last_collection_total,
42
	a.last_collection_date,
43
	a.last_collection_mdid,
44
	a.last_aggregation_total,
45
	a.last_aggregation_date,
46
	a.last_aggregation_mdid,
47
	a.last_download_total,
48
	a.last_download_date,
49
	a.last_download_objid,
50
	a.last_validation_job
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/resources/eu/dnetlib/enabling/datasources/queries/setActive.sql
1
UPDATE dsm_api SET active = :active WHERE id = :apiId AND datasource = :dsId
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/resources/eu/dnetlib/enabling/datasources/queries/searchDsByType.sql
1
SELECT
2
	ds.id           AS id,
3
	ds.officialname AS name,
4
	ds.typology     AS typology,
5
	array_agg(a.id) AS apis
6
FROM
7
	dsm_datasources ds
8
	LEFT OUTER JOIN dsm_api a ON (ds.id = a.datasource)
9
WHERE
10
	ds.typology = :type
11
GROUP BY
12
	ds.id, ds.officialname, ds.typology
13
ORDER BY
14
	 ds.officialname
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/resources/eu/dnetlib/enabling/datasources/queries/getDatasource.sql
1
SELECT
2
	ds.id                        AS "id",
3
	ds.officialname              AS "officialname",
4
	ds.englishname               AS "englishname",
5
	ds.websiteurl                AS "websiteurl",
6
	ds.logourl                   AS "logourl",
7
	ds.contactemail              AS "contactemail",
8
	ds.latitude                  AS "latitude",
9
	ds.longitude                 AS "longitude",
10
	ds.timezone                  AS "timezone",
11
	ds.namespaceprefix           AS "namespaceprefix",
12
	ds.languages                 AS "languages",
13
	ds.od_contenttypes           AS "od_contenttypes",
14
	ds.collectedfrom             AS "collectedfrom",
15
	ds.dateofvalidation          AS "dateofvalidation",
16
	ds.typology                  AS "typology",
17
	ds.provenanceaction          AS "provenanceaction",
18
	ds.dateofcollection          AS "dateofcollection",
19
	ds.platform                  AS "platform",
20
	ds.activationid              AS "activationId",
21
	ds.description               AS "description",
22
	ds.releasestartdate          AS "releasestartdate",
23
	ds.releaseenddate            AS "releaseenddate",
24
	ds.missionstatementurl       AS "missionstatementurl",
25
	ds.dataprovider              AS "dataprovider",
26
	ds.serviceprovider           AS "serviceprovider",
27
	ds.databaseaccesstype        AS "databaseaccesstype",
28
	ds.datauploadtype            AS "datauploadtype",
29
	ds.databaseaccessrestriction AS "databaseaccessrestriction",
30
	ds.datauploadrestriction     AS "datauploadrestriction",
31
	ds.versioning                AS "versioning",
32
	ds.citationguidelineurl      AS "citationguidelineurl",
33
	ds.qualitymanagementkind     AS "qualitymanagementkind",
34
	ds.pidsystems                AS "pidsystems",
35
	ds.certificates              AS "certificates",
36
	ds.aggregator                AS "aggregator",
37
	ds.issn                      AS "issn",
38
	ds.eissn                     AS "eissn",
39
	ds.lissn                     AS "lissn",
40
	ds.registeredby              AS "registeredby",
41
	ds.subjects                  AS "subjects",
42
	ds.managed                   AS "managed"
43
FROM
44
	dsm_datasources ds
45
WHERE
46
	ds.id = :dsId
47
	
48

  
49
	
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/resources/eu/dnetlib/enabling/datasources/queries/searchApisNormal.sql
1
SELECT 
2
	a.id                                                                     AS "id",
3
	a.protocol                                                               AS "protocol",
4
	coalesce(a.compatibility_override, a.compatibility)                      AS "compliance",
5
	a.active                                                                 AS "active",
6
	a.baseurl                                                                AS "baseUrl",
7
	coalesce(a.last_aggregation_date::text, a.last_download_date::text, '')  AS "aggrDate",
8
	coalesce(a.last_aggregation_total, a.last_download_total, 0)             AS "aggrTotal",
9
	ds.id                                                                    AS "dsId",
10
	ds.officialname                                                          AS "name",
11
	ds.englishname                                                           AS "alternativeName",
12
	ds.namespaceprefix                                                       AS "prefix",
13
	ds.websiteurl                                                            AS "websiteUrl",
14
	coalesce((array_agg(o.country))[1], '-')                                 AS "country",
15
    (array_agg(o.legalname))[1]                                              AS "organization"
16
FROM dsm_api a
17
	LEFT OUTER JOIN dsm_datasources ds ON (a.datasource = ds.id)
18
	LEFT OUTER JOIN dsm_datasource_organization dsorg ON (ds.id = dsorg.datasource)
19
	LEFT OUTER JOIN dsm_organizations o ON (dsorg.organization = o.id)
20
WHERE
21
	ds.id                 ILIKE :value 
22
	OR ds.officialname    ILIKE :value
23
	OR ds.englishname     ILIKE :value
24
	OR ds.namespaceprefix ILIKE :value
25
	OR ds.websiteurl      ILIKE :value
26
	OR a.baseurl          ILIKE :value
27
	OR o.legalname        ILIKE :value
28
GROUP BY
29
	a.id,
30
	a.protocol,
31
	a.compatibility_override,
32
	a.compatibility,
33
	a.active,
34
	a.last_aggregation_date,
35
	a.last_download_date,
36
	a.last_aggregation_total,
37
	a.last_download_total,
38
	ds.id,
39
	ds.officialname,
40
	ds.englishname,
41
	ds.namespaceprefix,
42
	ds.websiteurl
43

  
44

  
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/resources/eu/dnetlib/enabling/datasources/queries/searchApisUsingField.sql
1
SELECT * FROM (SELECT 
2
	a.id                                                                     AS "id",
3
	a.protocol                                                               AS "protocol",
4
	coalesce(a.compatibility_override, a.compatibility)                      AS "compliance",
5
	a.active                                                                 AS "active",
6
	a.baseurl                                                                AS "baseUrl",
7
	a.typology                                                               AS "type",
8
	coalesce(a.last_aggregation_date::text, a.last_download_date::text, '')  AS "aggrDate",
9
	coalesce(a.last_aggregation_total, a.last_download_total, 0)             AS "aggrTotal",
10
	ds.id                                                                    AS "dsId",
11
	ds.officialname                                                          AS "name",
12
	ds.englishname                                                           AS "alternativeName",
13
	ds.namespaceprefix                                                       AS "prefix",
14
	ds.websiteurl                                                            AS "websiteUrl",
15
	coalesce((array_agg(o.country))[1], '-')                                 AS "country",
16
    (array_agg(o.legalname))[1]                                              AS "organization"
17
FROM dsm_api a
18
	LEFT OUTER JOIN dsm_datasources ds ON (a.datasource = ds.id)
19
	LEFT OUTER JOIN dsm_datasource_organization dsorg ON (ds.id = dsorg.datasource)
20
	LEFT OUTER JOIN dsm_organizations o ON (dsorg.organization = o.id)
21
GROUP BY
22
	a.id,
23
	a.protocol,
24
	a.compatibility_override,
25
	a.compatibility,
26
	a.active,
27
	a.last_aggregation_date,
28
	a.last_download_date,
29
	a.last_aggregation_total,
30
	a.last_download_total,
31
	ds.id,
32
	ds.officialname,
33
	ds.englishname,
34
	ds.namespaceprefix,
35
	ds.websiteurl
36
) AS t WHERE
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/resources/eu/dnetlib/enabling/datasources/queries/insertApi.sql
1
INSERT INTO dsm_api(
2
	id,
3
	protocol, 
4
	baseurl,
5
	datasource,
6
	contentdescription,
7
	typology,
8
	compatibility,
9
	metadata_identifier_path,
10
	removable
11
) VALUES (
12
	:apiId,
13
	:protocol, 
14
	:baseUrl,
15
	:dsId,
16
	:contentDescription,
17
	:typology,
18
	:compatibility,
19
	:metadataIdentifierPath,
20
	:removable
21
)
modules/dnet-openaire-datasource-manager/tags/dnet-openaire-datasource-manager-1.0.6/src/main/resources/eu/dnetlib/enabling/datasources/queries/addDatasource.sql
1
INSERT INTO dsm_datasources(
2
	id,
3
	officialname,
4
	englishname,
5
	websiteurl,
6
	logourl,
7
	contactemail,
8
	latitude,
9
	longitude,
10
	timezone,
11
	namespaceprefix,
12
	languages,--array
13
	od_contenttypes,
14
	collectedfrom,
15
	dateofvalidation,
16
	typology,
17
	provenanceaction,
18
	platform,
19
	activationid,
20
	description,
21
	releasestartdate,
22
	releaseenddate,
23
	missionstatementurl,
24
	dataprovider,
25
	serviceprovider,
26
	databaseaccesstype,
27
	datauploadtype,
28
	databaseaccessrestriction,
29
	datauploadrestriction,
30
	versioning,
31
	citationguidelineurl,
32
	qualitymanagementkind,
33
	pidsystems,
34
	certificates,
35
	aggregator,
36
	issn,
37
	eissn,
38
	lissn,
39
	registeredby,
40
	subjects,--array
41
	managed
42
) VALUES (
43
	:id,
44
	:officialname,
45
	:englishname,
46
	:websiteurl,
47
	:logourl,
48
	:contactemail,
49
	:latitude,
50
	:longitude,
51
	:timezone,
52
	:namespaceprefix,
53
	:languages,--array
54
	:od_contenttypes,
55
	:collectedfrom,
56
	:dateofvalidation,
57
	:typology,
58
	:provenanceaction,
59
	:platform,
60
	:activationid,
61
	:description,
62
	:releasestartdate,
63
	:releaseenddate,
64
	:missionstatementurl,
65
	:dataprovider,
66
	:serviceprovider,
67
	:databaseaccesstype,
68
	:datauploadtype,
69
	:databaseaccessrestriction,
... This diff was truncated because it exceeds the maximum size that can be displayed.

Also available in: Unified diff