|
1
|
package eu.dnetlib.repo.manager.service.utils;
|
|
2
|
|
|
3
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
|
4
|
import com.fasterxml.jackson.core.type.TypeReference;
|
|
5
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
6
|
import eu.dnetlib.domain.data.Repository;
|
|
7
|
import eu.dnetlib.domain.data.RepositoryInterface;
|
|
8
|
import eu.dnetlib.repo.manager.service.controllers.RepositoryApi;
|
|
9
|
import eu.dnetlib.repo.manager.shared.*;
|
|
10
|
import org.apache.log4j.Logger;
|
|
11
|
import org.json.JSONArray;
|
|
12
|
import org.json.JSONException;
|
|
13
|
import org.json.JSONObject;
|
|
14
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
15
|
|
|
16
|
import java.io.*;
|
|
17
|
import java.text.ParseException;
|
|
18
|
import java.text.SimpleDateFormat;
|
|
19
|
import java.util.*;
|
|
20
|
|
|
21
|
public class Converter {
|
|
22
|
|
|
23
|
private static final Logger LOGGER = Logger.getLogger(Converter.class);
|
|
24
|
|
|
25
|
public static Repository jsonToRepositoryObject(JSONObject repositoryObject) throws JSONException {
|
|
26
|
|
|
27
|
Repository repository = new Repository();
|
|
28
|
|
|
29
|
if( repositoryObject.get("datasource").equals(null))
|
|
30
|
return null;
|
|
31
|
|
|
32
|
JSONObject datasource = (JSONObject) repositoryObject.get("datasource");
|
|
33
|
|
|
34
|
repository.setActivationId(datasource.get("activationId").toString());
|
|
35
|
repository.setAggregator(datasource.get("aggregator").toString());
|
|
36
|
repository.setCertificates(datasource.get("certificates").toString());
|
|
37
|
repository.setCitationGuidelineUrl(datasource.get("citationguidelineurl").toString());
|
|
38
|
repository.setCollectedFrom( datasource.get("collectedfrom").toString());
|
|
39
|
|
|
40
|
repository.setContactEmail(datasource.get("contactemail").toString());
|
|
41
|
if(repository.getContactEmail().equals("null"))
|
|
42
|
repository.setContactEmail("");
|
|
43
|
|
|
44
|
repository.setDatabaseAccessRestriction(datasource.get("databaseaccessrestriction").toString());
|
|
45
|
repository.setDatabaseAccessType(datasource.get("databaseaccesstype").toString());
|
|
46
|
repository.setDataUploadRestriction(datasource.get("datauploadrestriction").toString());
|
|
47
|
repository.setDataUploadType(datasource.get("datauploadtype").toString());
|
|
48
|
repository.setDateOfCollection(convertStringToDate( datasource.get("dateofcollection").toString()));
|
|
49
|
repository.setDateOfValidation(convertStringToDate( datasource.get("dateofvalidation").toString()));
|
|
50
|
|
|
51
|
repository.setDescription(datasource.get("description").toString());
|
|
52
|
if(repository.getDescription().equals("null"))
|
|
53
|
repository.setDescription("");
|
|
54
|
|
|
55
|
repository.setEissn(datasource.get("eissn").toString());
|
|
56
|
|
|
57
|
repository.setEnglishName( datasource.get("englishname").toString());
|
|
58
|
if(repository.getEnglishName().equals("null"))
|
|
59
|
repository.setEnglishName("");
|
|
60
|
|
|
61
|
|
|
62
|
repository.setId(datasource.get("id").toString());
|
|
63
|
repository.setIssn(datasource.get("issn").toString());
|
|
64
|
repository.setOdLanguages(datasource.get("languages").toString());
|
|
65
|
repository.setLatitude( toDouble(datasource.get("latitude").toString()));
|
|
66
|
repository.setLissn(datasource.get("lissn").toString());
|
|
67
|
|
|
68
|
repository.setLogoUrl(datasource.get("logourl").toString());
|
|
69
|
if(repository.getLogoUrl().equals("null"))
|
|
70
|
repository.setLogoUrl("");
|
|
71
|
|
|
72
|
repository.setLongitude(toDouble(datasource.get("longitude").toString()));
|
|
73
|
//datasource.get("managed");
|
|
74
|
repository.setMissionStatementUrl(datasource.get("missionstatementurl").toString());
|
|
75
|
repository.setNamespacePrefix(datasource.get("namespaceprefix").toString());
|
|
76
|
repository.setOdContentTypes(datasource.get("od_contenttypes").toString());
|
|
77
|
repository.setOfficialName(datasource.get("officialname").toString());
|
|
78
|
if(repository.getOfficialName().equals("null"))
|
|
79
|
repository.setOfficialName("");
|
|
80
|
|
|
81
|
repository.setPidSystems(datasource.get("pidsystems").toString());
|
|
82
|
//datasource.get("platform");
|
|
83
|
repository.setProvenanceActionClass( datasource.get("provenanceaction").toString());
|
|
84
|
repository.setQualityManagementKind(datasource.get("qualitymanagementkind").toString());
|
|
85
|
repository.setRegisteredBy(datasource.get("registeredby").toString());
|
|
86
|
repository.setReleaseEndDate(convertStringToDate(datasource.get("releaseenddate").toString()));
|
|
87
|
repository.setReleaseStartDate(convertStringToDate(datasource.get("releasestartdate").toString()));
|
|
88
|
repository.setServiceProvider(Boolean.valueOf(datasource.get("serviceprovider").toString()));
|
|
89
|
//datasource.get("subjects");
|
|
90
|
Double timezone = toDouble(datasource.get("timezone").toString());
|
|
91
|
repository.setTimezone(timezone!=null?timezone:0.0);
|
|
92
|
repository.setTypology(datasource.get("platform").toString());
|
|
93
|
repository.setVersioning(Boolean.valueOf(datasource.get("versioning").toString()));
|
|
94
|
repository.setWebsiteUrl(datasource.get("websiteurl").toString());
|
|
95
|
repository.setDatasourceClass(datasource.get("typology").toString());
|
|
96
|
|
|
97
|
//TODO change organization to list
|
|
98
|
repository.setOrganization( ((JSONArray)datasource.get("organizations")).getJSONObject(0).get("legalname").toString());
|
|
99
|
String countryCode = ((JSONArray)datasource.get("organizations")).getJSONObject(0).get("country").toString();
|
|
100
|
repository.setCountryCode(countryCode);
|
|
101
|
|
|
102
|
|
|
103
|
String collectedFrom = datasource.get("collectedfrom").toString();
|
|
104
|
//TODO check data consistency
|
|
105
|
String type = "UNKNOWN";
|
|
106
|
if (collectedFrom.equalsIgnoreCase("openaire____::opendoar")) {
|
|
107
|
type = "opendoar";
|
|
108
|
} else if (collectedFrom.equalsIgnoreCase("openaire____::re3data")) {
|
|
109
|
type = "re3data";
|
|
110
|
} else if (collectedFrom.equalsIgnoreCase("infrastruct_::openaire")) {
|
|
111
|
type = "journal";
|
|
112
|
}
|
|
113
|
|
|
114
|
repository.setDatasourceType(type);
|
|
115
|
|
|
116
|
|
|
117
|
return repository;
|
|
118
|
}
|
|
119
|
|
|
120
|
|
|
121
|
|
|
122
|
public static Date convertStringToDate(String date){
|
|
123
|
|
|
124
|
if(Objects.equals(date, "null"))
|
|
125
|
return null;
|
|
126
|
|
|
127
|
SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
|
|
128
|
try {
|
|
129
|
return formatter.parse(date);
|
|
130
|
} catch (ParseException e) {
|
|
131
|
e.printStackTrace();
|
|
132
|
}
|
|
133
|
return null;
|
|
134
|
}
|
|
135
|
|
|
136
|
public static String convertDateToString(Date date){
|
|
137
|
|
|
138
|
if(Objects.equals(date, null))
|
|
139
|
return null;
|
|
140
|
|
|
141
|
SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
|
|
142
|
return formatter.format(date);
|
|
143
|
}
|
|
144
|
|
|
145
|
public static Double toDouble(String number){
|
|
146
|
if(Objects.equals(number, "null"))
|
|
147
|
return 0.0;
|
|
148
|
else
|
|
149
|
return Double.valueOf(number);
|
|
150
|
}
|
|
151
|
|
|
152
|
public static List<Repository> jsonToRepositoryList(JSONArray rs) throws JSONException {
|
|
153
|
|
|
154
|
List<Repository> resultSet = new ArrayList<>();
|
|
155
|
for(int i=0;i<rs.length();i++)
|
|
156
|
resultSet.add(jsonToRepositoryObject( rs.getJSONObject(i)) );
|
|
157
|
return resultSet;
|
|
158
|
}
|
|
159
|
|
|
160
|
public static List<RepositoryInterface> jsonToRepositoryInterfaceList(JSONArray rs) throws JSONException {
|
|
161
|
|
|
162
|
List<RepositoryInterface> resultSet = new ArrayList<>();
|
|
163
|
for(int i=0;i<rs.length();i++)
|
|
164
|
resultSet.add(jsonToRepositoryInterfaceObject( rs.getJSONObject(i)) );
|
|
165
|
return resultSet;
|
|
166
|
}
|
|
167
|
|
|
168
|
public static RepositoryInterface jsonToRepositoryInterfaceObject(JSONObject repositoryInterfaceObject) throws JSONException {
|
|
169
|
|
|
170
|
RepositoryInterface repositoryInterface = new RepositoryInterface();
|
|
171
|
|
|
172
|
repositoryInterface.setBaseUrl(repositoryInterfaceObject.get("baseurl").toString());
|
|
173
|
repositoryInterface.setContentDescription(repositoryInterfaceObject.get("contentdescription").toString());
|
|
174
|
repositoryInterface.setId(repositoryInterfaceObject.get("id").toString());
|
|
175
|
repositoryInterface.setMetadataIdentifierPath(repositoryInterfaceObject.get("metadataIdentifierPath").toString());
|
|
176
|
repositoryInterface.setAccessProtocol(repositoryInterfaceObject.get("protocol").toString());
|
|
177
|
repositoryInterface.setTypology(repositoryInterfaceObject.get("typology").toString());
|
|
178
|
repositoryInterface.setDesiredCompatibilityLevel(repositoryInterfaceObject.get("compatibility").toString());
|
|
179
|
repositoryInterface.setActive(Boolean.parseBoolean(repositoryInterfaceObject.get("active").toString()));
|
|
180
|
repositoryInterface.setRemovable(Boolean.parseBoolean(repositoryInterfaceObject.get("removable").toString()));
|
|
181
|
repositoryInterface.setCompliance(repositoryInterfaceObject.get("compatibility").toString());
|
|
182
|
|
|
183
|
Map<String, String> accessParams = new HashMap<>();
|
|
184
|
Map<String, String> extraFields = new HashMap<>();
|
|
185
|
|
|
186
|
ObjectMapper mapper = new ObjectMapper();
|
|
187
|
JSONArray apiparams = repositoryInterfaceObject.getJSONArray("apiparam");
|
|
188
|
|
|
189
|
for(int i=0;i<apiparams.length();i++)
|
|
190
|
accessParams.put(apiparams.getJSONObject(i).getString("param"),apiparams.getJSONObject(i).getString("value"));
|
|
191
|
|
|
192
|
return repositoryInterface;
|
|
193
|
}
|
|
194
|
|
|
195
|
public static String repositoryObjectToJson(Repository repository) throws JSONException {
|
|
196
|
|
|
197
|
JSONObject jsonObject = new JSONObject();
|
|
198
|
jsonObject.put("activationId",repository.getActivationId());
|
|
199
|
jsonObject.put("aggregator",repository.getAggregator());
|
|
200
|
jsonObject.put("certificates",repository.getCertificates());
|
|
201
|
jsonObject.put("citationguidelineurl",repository.getCitationGuidelineUrl());
|
|
202
|
jsonObject.put("collectedfrom",repository.getCollectedFrom());
|
|
203
|
jsonObject.put("contactemail",repository.getContactEmail());
|
|
204
|
jsonObject.put("databaseaccessrestriction",repository.getDatabaseAccessRestriction());
|
|
205
|
jsonObject.put("databaseaccesstype",repository.getDatabaseAccessType());
|
|
206
|
jsonObject.put("datauploadrestriction",repository.getDataUploadRestriction());
|
|
207
|
jsonObject.put("datauploadtype",repository.getDataUploadType());
|
|
208
|
jsonObject.put("dateofcollection",convertDateToString(repository.getDateOfCollection()));
|
|
209
|
jsonObject.put("dateofvalidation",convertDateToString(repository.getDateOfValidation()));
|
|
210
|
jsonObject.put("description",repository.getDescription());
|
|
211
|
jsonObject.put("eissn",repository.getEissn());
|
|
212
|
jsonObject.put("englishname",repository.getEnglishName());
|
|
213
|
jsonObject.put("id",repository.getId());
|
|
214
|
jsonObject.put("issn",repository.getIssn());
|
|
215
|
jsonObject.put("languages",repository.getOdLanguages());
|
|
216
|
jsonObject.put("latitude",repository.getLatitude().toString());
|
|
217
|
jsonObject.put("lissn",repository.getLissn());
|
|
218
|
jsonObject.put("logourl",repository.getLogoUrl());
|
|
219
|
jsonObject.put("longitude",repository.getLongitude().toString());
|
|
220
|
jsonObject.put("missionstatementurl",repository.getMissionStatementUrl());
|
|
221
|
jsonObject.put("namespaceprefix",repository.getNamespacePrefix());
|
|
222
|
jsonObject.put("od_contenttypes",repository.getOdContentTypes());
|
|
223
|
jsonObject.put("officialname",repository.getOfficialName());
|
|
224
|
jsonObject.put("pidsystems",repository.getPidSystems());
|
|
225
|
jsonObject.put("provenanceaction",repository.getProvenanceActionClass());
|
|
226
|
jsonObject.put("qualitymanagementkind",repository.getQualityManagementKind());
|
|
227
|
jsonObject.put("registeredby",repository.getRegisteredBy());
|
|
228
|
jsonObject.put("releaseenddate",convertDateToString(repository.getReleaseEndDate()));
|
|
229
|
jsonObject.put("releasestartdate",convertDateToString(repository.getReleaseStartDate()));
|
|
230
|
jsonObject.put("serviceprovider",repository.getServiceProvider());
|
|
231
|
jsonObject.put("timezone",repository.getTimezone());
|
|
232
|
jsonObject.put("typology",repository.getTypology());
|
|
233
|
jsonObject.put("versioning",repository.getVersioning());
|
|
234
|
jsonObject.put("websiteurl",repository.getWebsiteUrl());
|
|
235
|
|
|
236
|
//datasource.get("managed");
|
|
237
|
//datasource.get("platform");
|
|
238
|
//datasource.get("subjects");
|
|
239
|
return jsonObject.toString();
|
|
240
|
}
|
|
241
|
|
|
242
|
public static String repositoryInterfaceObjectToJson(Repository repository,RepositoryInterface repositoryInterface) throws JSONException {
|
|
243
|
|
|
244
|
JSONObject jsonObject = new JSONObject();
|
|
245
|
|
|
246
|
jsonObject.put("baseurl",repositoryInterface.getBaseUrl());
|
|
247
|
jsonObject.put("contentdescription",repositoryInterface.getContentDescription());
|
|
248
|
jsonObject.put("id",repositoryInterface.getId());
|
|
249
|
jsonObject.put("metadataIdentifierPath",repositoryInterface.getMetadataIdentifierPath());
|
|
250
|
jsonObject.put("protocol",repositoryInterface.getAccessProtocol());
|
|
251
|
jsonObject.put("typology",repositoryInterface.getTypology());
|
|
252
|
jsonObject.put("compatibility",repositoryInterface.getDesiredCompatibilityLevel());
|
|
253
|
jsonObject.put("datasource",repository.getId());
|
|
254
|
jsonObject.put("metadataIdentifierPath",repositoryInterface.getMetadataIdentifierPath());
|
|
255
|
jsonObject.put("protocol",repositoryInterface.getAccessProtocol());
|
|
256
|
jsonObject.put("removable",repositoryInterface.isRemovable());
|
|
257
|
jsonObject.put("active",repositoryInterface.isActive());
|
|
258
|
|
|
259
|
|
|
260
|
JSONArray apiparams = new JSONArray();
|
|
261
|
for(String param: repositoryInterface.getAccessParams().keySet()){
|
|
262
|
JSONObject jo = new JSONObject();
|
|
263
|
jo.put("param",param);
|
|
264
|
jo.put("value",repositoryInterface.getAccessParams().get(param));
|
|
265
|
apiparams.put(jo);
|
|
266
|
}
|
|
267
|
jsonObject.put("apiparam",apiparams);
|
|
268
|
|
|
269
|
|
|
270
|
jsonObject.put("lastCollectionDate",repositoryInterface.getLastCollectionDate());
|
|
271
|
//jsonObject.put("lastCollectionMdid",repositoryInterface);
|
|
272
|
//jsonObject.put("lastCollectionTotal");
|
|
273
|
//jsonObject.put("lastDownloadDate");
|
|
274
|
// jsonObject.put("lastDownloadMdid");
|
|
275
|
// jsonObject.put("lastDownloadTotal");
|
|
276
|
// jsonObject.put("lastValidationJob");
|
|
277
|
//jsonObject.put("lastAggregationDate");
|
|
278
|
//jsonObject.put("lastAggregationMdid");
|
|
279
|
//jsonObject.put("lastAggregationTotal");
|
|
280
|
|
|
281
|
return jsonObject.toString();
|
|
282
|
}
|
|
283
|
|
|
284
|
public static ArrayList<String> readFile(String filename) {
|
|
285
|
String line;
|
|
286
|
ArrayList<String> list = new ArrayList<String>();
|
|
287
|
try {
|
|
288
|
//InputStream in = Converter.class.getResourceAsStream("resources/eu/dnetlib/repo/manager/service/utils/"+filename);
|
|
289
|
InputStream in = Converter.class.getClass().getResourceAsStream("/eu/**/" + filename);
|
|
290
|
BufferedReader br = new BufferedReader(new InputStreamReader(in));
|
|
291
|
while((line = br.readLine()) != null) {
|
|
292
|
list.add(line.trim());
|
|
293
|
}
|
|
294
|
br.close();
|
|
295
|
} catch (IOException e) {
|
|
296
|
LOGGER.debug("Error opening file!");
|
|
297
|
e.printStackTrace();
|
|
298
|
}
|
|
299
|
return list;
|
|
300
|
}
|
|
301
|
|
|
302
|
public static List<AggregationDetails> getAggregationHistoryFromJson(JSONObject repositoryObject) throws JSONException {
|
|
303
|
|
|
304
|
if( repositoryObject.get("aggregationHistory").toString().equals("[]") ||
|
|
305
|
repositoryObject.get("aggregationHistory")!= null)
|
|
306
|
return null;
|
|
307
|
|
|
308
|
|
|
309
|
JSONArray rs = new JSONArray(repositoryObject.get("aggregationHistory").toString());
|
|
310
|
List<AggregationDetails> aggregationDetailsList = new ArrayList<>();
|
|
311
|
for(int i=0;i<rs.length();i++)
|
|
312
|
aggregationDetailsList.add(jsonToAggregationDetails(rs.getJSONObject(i)));
|
|
313
|
return aggregationDetailsList;
|
|
314
|
}
|
|
315
|
|
|
316
|
private static AggregationDetails jsonToAggregationDetails(JSONObject aggregationObject) throws JSONException {
|
|
317
|
|
|
318
|
AggregationDetails aggregationDetails = new AggregationDetails();
|
|
319
|
aggregationDetails.setAggregationStage(aggregationObject.get("aggregationStage").toString());
|
|
320
|
//aggregationDetails.setCollectionMode(aggregationObject.get("collectionMode").toString());
|
|
321
|
aggregationDetails.setDate(convertStringToDate(aggregationObject.get("date").toString()));
|
|
322
|
aggregationDetails.setNumberOfRecords(Integer.parseInt(aggregationObject.get("numberOfRecords").toString()));
|
|
323
|
return aggregationDetails;
|
|
324
|
}
|
|
325
|
|
|
326
|
public static AggregationDetails getLastCollectionFromJson(JSONObject repositoryObject) throws JSONException {
|
|
327
|
|
|
328
|
if( repositoryObject.get("lastCollection").equals(null))
|
|
329
|
return null;
|
|
330
|
|
|
331
|
return jsonToAggregationDetails(repositoryObject.getJSONObject("lastCollection"));
|
|
332
|
}
|
|
333
|
|
|
334
|
public static AggregationDetails getLastTransformationFromJson(JSONObject repositoryObject) throws JSONException {
|
|
335
|
|
|
336
|
if( repositoryObject.get("lastTransformation").equals(null))
|
|
337
|
return null;
|
|
338
|
|
|
339
|
return jsonToAggregationDetails(repositoryObject.getJSONObject("lastTransformation"));
|
|
340
|
}
|
|
341
|
|
|
342
|
public static List<Timezone> toTimezones(List<String> timezones) {
|
|
343
|
|
|
344
|
List<Timezone> tmz = new ArrayList<>();
|
|
345
|
for(String t : timezones){
|
|
346
|
String[] s = t.split("\t");
|
|
347
|
tmz.add(new Timezone(s[1],Double.parseDouble(s[0])));
|
|
348
|
}
|
|
349
|
return tmz;
|
|
350
|
}
|
|
351
|
}
|