Project

General

Profile

1
package eu.dnetlib.enabling.datasources;
2

    
3
import java.io.IOException;
4
import java.sql.Array;
5
import java.sql.Date;
6
import java.sql.SQLException;
7
import java.sql.Timestamp;
8
import java.util.*;
9
import java.util.stream.Collectors;
10

    
11
import eu.dnetlib.enabling.datasources.common.*;
12
import eu.dnetlib.miscutils.datetime.DateUtils;
13
import org.apache.commons.lang.BooleanUtils;
14
import org.apache.commons.lang.StringUtils;
15
import org.apache.commons.logging.Log;
16
import org.apache.commons.logging.LogFactory;
17
import org.dom4j.Document;
18
import org.dom4j.DocumentException;
19
import org.dom4j.Element;
20
import org.dom4j.io.SAXReader;
21
import org.springframework.core.io.ClassPathResource;
22
import org.springframework.core.io.Resource;
23

    
24
public class DatasourceFunctions {
25

    
26
	private static final Log log = LogFactory.getLog(DatasourceFunctions.class);
27

    
28
	private static final Resource baseDsProfile = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/datasource_base.xml");
29

    
30
	public static SimpleDatasource mapToSimpleDs(final Map<String, Object> map) {
31
		final SimpleDatasource ds = new SimpleDatasource();
32
		ds.setId(castObject(map.get("id"), String.class));
33
		ds.setName(castObject(map.get("name"), String.class));
34
		ds.setOrigId(castObject(map.get("id"), String.class));
35
		ds.setTypology(castObject(map.get("typology"), String.class));
36
		ds.setValid(true);
37
		try {
38
			final Array arr = castObject(map.get("apis"), Array.class);
39
			if (arr != null) {
40
				ds.setApis(Arrays.stream((Object[]) arr.getArray())
41
						.filter(Objects::nonNull)
42
						.map(Object::toString)
43
						.collect(Collectors.toSet()));
44
			}
45
		} catch (final SQLException e) {
46
			log.error("Error parsing array (apis)", e);
47
			throw new RuntimeException("Error parsing array (apis)", e);
48
		}
49

    
50
		return ds;
51
	}
52

    
53
	public static Datasource<Organization<?>, Identity> mapToDatasource(final Map<String, Object> map) {
54
		final Datasource<Organization<?>, Identity> ds = new Datasource<>();
55
		ds.setId(castObject(map.get("id"), String.class));
56
		ds.setOfficialname(castObject(map.get("officialname"), String.class));
57
		ds.setEnglishname(castObject(map.get("englishname"), String.class));
58
		ds.setWebsiteurl(castObject(map.get("websiteurl"), String.class));
59
		ds.setLogourl(castObject(map.get("logourl"), String.class));
60
		ds.setContactemail(castObject(map.get("contactemail"), String.class));
61
		ds.setLatitude(castObject(map.get("latitude"), Double.class));
62
		ds.setLongitude(castObject(map.get("longitude"), Double.class));
63
		ds.setTimezone(castObject(map.get("timezone"), String.class));
64
		ds.setNamespaceprefix(castObject(map.get("namespaceprefix"), String.class));
65
		ds.setLanguages(castObject(map.get("languages"), String.class));
66
		ds.setOd_contenttypes(castObject(map.get("od_contenttypes"), String.class));
67
		ds.setCollectedfrom(castObject(map.get("collectedfrom"), String.class));
68
		ds.setDateofvalidation(castObject(map.get("dateofvalidation"), Date.class));
69
		ds.setTypology(castObject(map.get("typology"), String.class));
70
		ds.setProvenanceaction(castObject(map.get("provenanceaction"), String.class));
71
		ds.setDateofcollection(castObject(map.get("dateofcollection"), Date.class));
72
		ds.setPlatform(castObject(map.get("platform"), String.class));
73
		ds.setActivationId(castObject(map.get("activationId"), String.class));
74
		ds.setDescription(castObject(map.get("description"), String.class));
75
		ds.setReleasestartdate(castObject(map.get("releasestartdate"), Date.class));
76
		ds.setReleaseenddate(castObject(map.get("releaseenddate"), Date.class));
77
		ds.setMissionstatementurl(castObject(map.get("missionstatementurl"), String.class));
78
		ds.setDataprovider(castObject(map.get("dataprovider"), Boolean.class));
79
		ds.setServiceprovider(castObject(map.get("serviceprovider"), Boolean.class));
80
		ds.setDatabaseaccesstype(castObject(map.get("databaseaccesstype"), String.class));
81
		ds.setDatauploadtype(castObject(map.get("datauploadtype"), String.class));
82
		ds.setDatabaseaccessrestriction(castObject(map.get("databaseaccessrestriction"), String.class));
83
		ds.setDatauploadrestriction(castObject(map.get("datauploadrestriction"), String.class));
84
		ds.setVersioning(castObject(map.get("versioning"), Boolean.class));
85
		ds.setCitationguidelineurl(castObject(map.get("citationguidelineurl"), String.class));
86
		ds.setQualitymanagementkind(castObject(map.get("qualitymanagementkind"), String.class));
87
		ds.setPidsystems(castObject(map.get("pidsystems"), String.class));
88
		ds.setCertificates(castObject(map.get("certificates"), String.class));
89
		ds.setAggregator(castObject(map.get("aggregator"), String.class));
90
		ds.setIssn(castObject(map.get("issn"), String.class));
91
		ds.setEissn(castObject(map.get("eissn"), String.class));
92
		ds.setLissn(castObject(map.get("lissn"), String.class));
93
		ds.setRegisteredby(castObject(map.get("registeredby"), String.class));
94
		ds.setSubjects(castObject(map.get("subjects"), String.class));
95
		ds.setManaged(castObject(map.get("managed"), Boolean.class));
96
		return ds;
97
	}
98

    
99
	public static Identity mapToDsIdentity(final Map<String, Object> map) {
100
		final Identity id = new Identity();
101
		id.setPid(castObject(map.get("pid"), String.class));
102
		id.setIssuertype(castObject(map.get("issuertype"), String.class));
103
		return id;
104
	}
105

    
106
	public static Organization<Datasource<?, ?>> mapToDsOrganization(final Map<String, Object> map) {
107
		final Organization<Datasource<?, ?>> org = new Organization<>();
108
		org.setId(castObject(map.get("id"), String.class));
109
		org.setLegalshortname(castObject(map.get("legalshortname"), String.class));
110
		org.setLegalname(castObject(map.get("legalname"), String.class));
111
		org.setWebsiteurl(castObject(map.get("websiteurl"), String.class));
112
		org.setLogourl(castObject(map.get("logourl"), String.class));
113
		org.setCountry(castObject(map.get("country"), String.class));
114
		org.setCollectedfrom(castObject(map.get("collectedfrom"), String.class));
115
		org.setDateofcollection(castObject(map.get("dateofcollection"), Date.class));
116
		org.setProvenanceaction(castObject(map.get("provenanceaction"), String.class));
117
		return org;
118
	}
119

    
120
	public static SearchApisEntry mapToSearchApisEntry(final Map<String, Object> map) {
121
		final SearchApisEntry a = new SearchApisEntry();
122
		a.setId(castObject(map.get("id"), String.class));
123
		a.setCompliance(castObject(map.get("compliance"), String.class));
124
		a.setActive(castObject(map.get("active"), Boolean.class));
125
		a.setRepoId(castObject(map.get("dsId"), String.class));
126
		a.setRepoName(castObject(map.get("name"), String.class));
127
		a.setRepoCountry(castObject(map.get("country"), String.class));
128
		a.setRepoPrefix(castObject(map.get("prefix"), String.class));
129
		a.setAggrDate(castObject(map.get("aggrDate"), String.class));
130
		a.setAggrTotal(castObject(map.get("aggrTotal"), Integer.class));
131
		a.setProtocol(castObject(map.get("protocol"), String.class));
132
		a.setAlternativeName(castObject(map.get("alternativeName"), String.class));
133
		a.setRepoOrganization(castObject(map.get("organization"), String.class));
134
		return a;
135
	}
136

    
137
	public static Api<ApiParam> mapToApi(final Map<String, Object> map) {
138
		final Api<ApiParam> a = new Api<>();
139
		a.setId(castObject(map.get("id"), String.class));
140
		a.setProtocol(castObject(map.get("protocol"), String.class));
141
		a.setDatasource(castObject(map.get("datasource"), String.class));
142
		a.setContentdescription(castObject(map.get("contentdescription"), String.class));
143
		a.setActive(castObject(map.get("active"), Boolean.class));
144
		a.setRemovable(castObject(map.get("removable"), Boolean.class));
145
		a.setTypology(castObject(map.get("typology"), String.class));
146
		a.setCompatibility(castObject(map.get("compatibility"), String.class));
147
		a.setCompatibilityOverrided(castObject(map.get("isCompatibilityOverrided"), Boolean.class));
148
		a.setMetadataIdentifierPath(castObject(map.get("metadataIdentifierPath"), String.class));
149
		a.setLastCollectionTotal(castObject(map.get("lastCollectionTotal"), Integer.class));
150
		a.setLastCollectionDate(castObject(map.get("lastCollectionDate"), Timestamp.class));
151
		a.setLastCollectionMdid(castObject(map.get("lastCollectionMdid"), String.class));
152
		a.setLastAggregationTotal(castObject(map.get("lastAggregationTotal"), Integer.class));
153
		a.setLastAggregationDate(castObject(map.get("lastAggregationDate"), Timestamp.class));
154
		a.setLastAggregationMdid(castObject(map.get("lastAggregationMdid"), String.class));
155
		a.setLastDownloadTotal(castObject(map.get("lastDownloadTotal"), Integer.class));
156
		a.setLastDownloadDate(castObject(map.get("lastDownloadDate"), Timestamp.class));
157
		a.setLastDownloadObjid(castObject(map.get("lastDownloadObjid"), String.class));
158
		a.setLastValidationJob(castObject(map.get("lastValidationJob"), String.class));
159
		a.setBaseurl(castObject(map.get("baseUrl"), String.class));
160
		try {
161
			final Array arr = castObject(map.get("params"), Array.class);
162
			if (arr != null) {
163
				a.setApiParams(Arrays.stream((Object[]) arr.getArray())
164
						.filter(Objects::nonNull)
165
						.map(Object::toString)
166
						.map(s -> {
167
							final ApiParam p = new ApiParamImpl();
168
							p.setParam(StringUtils.substringBefore(s, "="));
169
							p.setValue(StringUtils.substringAfter(s, "="));
170
							return p;
171
						})
172
						.collect(Collectors.toSet()));
173
			}
174
		} catch (final SQLException e) {
175
			log.error("Error parsing array params", e);
176
			throw new RuntimeException("Error parsing array params", e);
177
		}
178

    
179
		return a;
180
	}
181

    
182
	public static String dsToProfile(final Datasource<Organization<?>, Identity> ds, final List<Api<ApiParam>> apis, final String profId)
183
			throws DocumentException, IOException {
184
		final Document doc = new SAXReader().read(baseDsProfile.getInputStream());
185

    
186
		setValue(doc, "//DATASOURCE_TYPE", ds.getPlatform());
187
		setValue(doc, "//DATASOURCE_ORIGINAL_ID", ds.getId());
188
		setValue(doc, "//TYPOLOGY", ds.getTypology());
189
		setValue(doc, "//OFFICIAL_NAME", ds.getOfficialname());
190
		setValue(doc, "//ENGLISH_NAME", ds.getEnglishname());
191
		setValue(doc, "//ICON_URI", ds.getLogourl());
192
		setValue(doc, "//COUNTRY", ds.getOrganizations().stream().map(Organization::getCountry).findFirst().orElse(""));
193
		setValue(doc, "//LONGITUDE", ds.getLongitude());
194
		setValue(doc, "//LATITUDE", ds.getLatitude());
195
		setValue(doc, "//TIMEZONE", ds.getTimezone());
196
		setValue(doc, "//REPOSITORY_WEBPAGE", ds.getWebsiteurl());
197
		setValue(doc, "//REPOSITORY_INSTITUTION", ds.getOrganizations().stream().map(Organization::getLegalname).findFirst().orElse(""));
198
		setValue(doc, "//ADMIN_INFO", ds.getContactemail());
199
		setValue(doc, "//REGISTERED_BY", ds.getRegisteredby());
200
		setValue(doc, "//LAST_UPDATE/@value", DateUtils.now_ISO8601());
201

    
202
		final Element ef = (Element) doc.selectSingleNode("//EXTRA_FIELDS");
203
		addExtraField(ef, "OpenAireDataSourceId", ds.getId());
204
		addExtraField(ef, "NamespacePrefix", ds.getNamespaceprefix());
205
		addExtraField(ef, "VERIFIED", "NO");
206
		addExtraField(ef, "aggregatorName", ds.getAggregator());
207
		addExtraField(ef, "dateOfValidation", ds.getDateofvalidation());
208
		addExtraField(ef, "dateOfCollection", ds.getDateofcollection());
209
		addExtraField(ef, "mergeHomonyms", "NO");
210
		addExtraField(ef, "ACTID", ds.getActivationId());
211

    
212
		if (apis != null) {
213
			final Element ifaces = (Element) doc.selectSingleNode("//INTERFACES");
214
			apis.forEach(api -> addInterface(ifaces, api));
215
		}
216
		return doc.asXML();
217
	}
218

    
219
	private static void setValue(final Document doc, final String xpath, final Object value) {
220
		if (value != null) {
221
			doc.selectSingleNode(xpath).setText(value.toString());
222
		}
223
	}
224

    
225
	private static void addInterface(final Element ifaces, final Api<?> api) {
226

    
227
		final Element ifc = ifaces.addElement("INTERFACE");
228
		ifc.addAttribute("id", api.getId());
229
		ifc.addAttribute("label", String.format("%s (%s)",
230
				StringUtils.defaultIfBlank(api.getTypology(), "-"),
231
				StringUtils.defaultIfBlank(api.getCompatibility(), "-")));
232
		ifc.addAttribute("typology", StringUtils.defaultIfBlank(api.getTypology(), ""));
233
		ifc.addAttribute("active", "" + BooleanUtils.toBooleanDefaultIfNull(api.getActive(), false));
234
		ifc.addAttribute("compliance", StringUtils.defaultIfBlank(api.getCompatibility(), ""));
235
		ifc.addAttribute("contentDescription", StringUtils.defaultIfBlank(api.getContentdescription(), ""));
236
		ifc.addAttribute("removable", "" + BooleanUtils.toBooleanDefaultIfNull(api.getRemovable(), false));
237

    
238
		final Element ap = ifc.addElement("ACCESS_PROTOCOL");
239
		api.getApiParams().forEach(p -> ap.addAttribute(p.getParam(), p.getValue()));
240
		ap.setText(StringUtils.defaultIfBlank(api.getProtocol(), ""));
241
		ifc.addElement("BASE_URL").setText(StringUtils.defaultIfBlank(api.getBaseurl(), ""));
242

    
243
		if (api.isCompatibilityOverrided()) {
244
			addInterfaceExtraField(ifc, "overriding_compliance", true);
245
		}
246

    
247
		addInterfaceExtraField(ifc, "metadata_identifier_path", api.getMetadataIdentifierPath());
248
		addInterfaceExtraField(ifc, "last_collection_date", api.getLastCollectionDate());
249
		addInterfaceExtraField(ifc, "last_collection_mdId", api.getLastCollectionMdid());
250
		addInterfaceExtraField(ifc, "last_collection_total", api.getLastCollectionTotal());
251
		addInterfaceExtraField(ifc, "last_aggregation_date", api.getLastAggregationDate());
252
		addInterfaceExtraField(ifc, "last_aggregation_mdId", api.getLastAggregationMdid());
253
		addInterfaceExtraField(ifc, "last_aggregation_total", api.getLastAggregationTotal());
254
		addInterfaceExtraField(ifc, "last_download_date", api.getLastDownloadDate());
255
		addInterfaceExtraField(ifc, "last_download_objId", api.getLastDownloadObjid());
256
		addInterfaceExtraField(ifc, "last_download_total", api.getLastDownloadTotal());
257
	}
258

    
259
	private static void addExtraField(final Element parent, final String field, final Object value) {
260
		if (value != null && StringUtils.isNotBlank(value.toString())) {
261
			final Element f = parent.addElement("FIELD");
262
			f.addElement("key").setText(field);
263
			f.addElement("value").setText(value.toString());
264
		}
265
	}
266

    
267
	private static void addInterfaceExtraField(final Element parent, final String field, final Object value) {
268
		if (value != null && StringUtils.isNotBlank(value.toString())) {
269
			final Element f = parent.addElement("INTERFACE_EXTRA_FIELD");
270
			f.addAttribute("name", field);
271
			f.setText(value.toString());
272
		}
273
	}
274

    
275
	public static Map<String, Object> dsToMap(final Datasource<Organization<?>, Identity> ds) {
276
		final Map<String, Object> map = new HashMap<>();
277

    
278
		map.put("id", ds.getId());
279
		map.put("officialname", ds.getOfficialname());
280
		map.put("englishname", ds.getEnglishname());
281
		map.put("websiteurl", ds.getWebsiteurl());
282
		map.put("logourl", ds.getLogourl());
283
		map.put("contactemail", ds.getContactemail());
284
		map.put("latitude", ds.getLatitude());
285
		map.put("longitude", ds.getLongitude());
286
		map.put("timezone", ds.getTimezone());
287
		map.put("namespaceprefix", ds.getNamespaceprefix());
288
		map.put("languages", ds.getLanguages());
289
		map.put("od_contenttypes", ds.getOd_contenttypes());
290
		map.put("collectedfrom", ds.getCollectedfrom());
291
		map.put("dateofvalidation", ds.getDateofvalidation());
292
		map.put("typology", ds.getTypology());
293
		map.put("provenanceaction", ds.getProvenanceaction());
294
		map.put("platform", ds.getPlatform());
295
		map.put("activationid", ds.getActivationId());
296
		map.put("description", ds.getDescription());
297
		map.put("releasestartdate", ds.getReleasestartdate());
298
		map.put("releaseenddate", ds.getReleaseenddate());
299
		map.put("missionstatementurl", ds.getMissionstatementurl());
300
		map.put("dataprovider", ds.isDataprovider());
301
		map.put("serviceprovider", ds.isServiceprovider());
302
		map.put("databaseaccesstype", ds.getDatabaseaccesstype());
303
		map.put("datauploadtype", ds.getDatauploadtype());
304
		map.put("databaseaccessrestriction", ds.getDatabaseaccessrestriction());
305
		map.put("datauploadrestriction", ds.getDatauploadrestriction());
306
		map.put("versioning", ds.isVersioning());
307
		map.put("citationguidelineurl", ds.getCitationguidelineurl());
308
		map.put("qualitymanagementkind", ds.getQualitymanagementkind());
309
		map.put("pidsystems", ds.getPidsystems());
310
		map.put("certificates", ds.getCertificates());
311
		map.put("aggregator", ds.getAggregator());
312
		map.put("issn", ds.getIssn());
313
		map.put("eissn", ds.getEissn());
314
		map.put("lissn", ds.getLissn());
315
		map.put("registeredby", ds.getRegisteredby());
316
		map.put("subjects", ds.getSubjects());
317
		map.put("managed", ds.isManaged());
318

    
319
		return map;
320
	}
321

    
322
	public static Map<String, Object> orgToMap(final String dsId, final Organization<?> org) {
323
		final Map<String, Object> map = new HashMap<>();
324
		map.put("dsId", dsId);
325
		map.put("orgId", org.getId());
326
		map.put("legalname", org.getLegalname());
327
		map.put("legalshortname", org.getLegalshortname());
328
		map.put("websiteurl", org.getWebsiteurl());
329
		map.put("logourl", org.getLogourl());
330
		map.put("country", org.getCountry());
331
		return map;
332
	}
333

    
334
	public static Map<String, Object> apiToMap(final Api<?> api) {
335
		final Map<String, Object> map = new HashMap<>();
336
		map.put("apiId", api.getId());
337
		map.put("protocol", api.getProtocol());
338
		map.put("baseUrl", api.getBaseurl());
339
		map.put("dsId", api.getDatasource());
340
		map.put("contentDescription", api.getContentdescription());
341
		map.put("typology", api.getTypology());
342
		map.put("compatibility", api.getCompatibility());
343
		map.put("metadataIdentifierPath", api.getMetadataIdentifierPath());
344
		// The other fields are not required in the INSERT operation
345
		return map;
346
	}
347

    
348
	@SuppressWarnings("unchecked")
349
	private static <T> T castObject(final Object o, final Class<T> clazz) {
350
		if (o == null) { return null; }
351
		if (clazz.isInstance(o)) { return (T) o; }
352
		throw new IllegalArgumentException("Type not managed: " + clazz.getSimpleName());
353
	}
354

    
355
}
(1-1/4)