Project

General

Profile

1
package eu.dnetlib.repo.manager.utils;
2

    
3
import com.fasterxml.jackson.core.JsonProcessingException;
4
import com.fasterxml.jackson.databind.ObjectMapper;
5
import eu.dnetlib.domain.data.Repository;
6
import eu.dnetlib.domain.data.RepositoryInterface;
7
import eu.dnetlib.repo.manager.domain.AggregationDetails;
8
import eu.dnetlib.repo.manager.domain.RepositorySnippet;
9
import eu.dnetlib.repo.manager.domain.Timezone;
10
import org.apache.commons.codec.digest.DigestUtils;
11
import org.apache.log4j.Logger;
12
import org.json.JSONArray;
13
import org.json.JSONException;
14
import org.json.JSONObject;
15
import org.springframework.stereotype.Component;
16

    
17
import java.io.BufferedReader;
18
import java.io.IOException;
19
import java.io.InputStream;
20
import java.io.InputStreamReader;
21
import java.text.ParseException;
22
import java.text.SimpleDateFormat;
23
import java.util.*;
24

    
25
@Component
26
public class Converter {
27

    
28
    private static final Logger LOGGER = Logger.getLogger(Converter.class);
29

    
30
    public static Repository jsonToRepositoryObject(JSONObject repositoryObject) throws JSONException {
31

    
32
        Repository repository = new Repository();
33

    
34
//        JSONObject datasource = repositoryObject.getJSONObject("datasource");
35
        JSONObject datasource = repositoryObject;
36

    
37
        //if( datasource.equals(null))
38
        //    return null;
39

    
40
        repository.setId(datasource.get("id").toString());
41
        repository.setOfficialName(datasource.get("officialname").toString());
42

    
43
        repository.setEnglishName(datasource.get("englishname").toString());
44
        if (repository.getEnglishName().equals("null"))
45
            repository.setEnglishName("");
46

    
47
        repository.setWebsiteUrl(datasource.get("websiteurl").toString());
48
        if (repository.getWebsiteUrl().equals("null"))
49
            repository.setWebsiteUrl("");
50

    
51
        repository.setLogoUrl(datasource.get("logourl").toString());
52
        if (repository.getLogoUrl().equals("null"))
53
            repository.setLogoUrl("");
54

    
55
        repository.setContactEmail(datasource.get("contactemail").toString());
56
        if (repository.getContactEmail().equals("null"))
57
            repository.setContactEmail("");
58

    
59

    
60
        repository.setLatitude(toDouble(datasource.get("latitude").toString()));
61
        repository.setLongitude(toDouble(datasource.get("longitude").toString()));
62
        Double timezone = toDouble(datasource.get("timezone").toString());
63
        repository.setTimezone(timezone != null ? timezone : 0.0);
64
        repository.setNamespacePrefix(datasource.get("namespaceprefix").toString());
65
        repository.setOdLanguages(datasource.get("languages").toString());
66
        repository.setDateOfValidation(convertStringToDate(datasource.get("dateofvalidation").toString()));
67

    
68
        /*  typology -> platform
69
         *  datasource class -> typology */
70
        repository.setTypology(datasource.get("platform").toString());
71
        if (repository.getTypology().equals("null"))
72
            repository.setTypology("");
73

    
74
//        // TODO: enable in future release
75
//        /* 07-04-2022 | "typology" -> "eoscDatasourceType" */
76
//        try { // FIXME: remove attemp to get typology if eoscDatasourceType fails
77
//            repository.setDatasourceClass(datasource.get("eoscDatasourceType").toString());
78
//        } catch (JSONException e) {
79
//            repository.setDatasourceClass(datasource.get("typology").toString());
80
//        }
81
        repository.setDatasourceClass(datasource.get("typology").toString());
82
//        <--
83

    
84
        repository.setDateOfCollection(convertStringToDate(datasource.get("dateofcollection").toString()));
85
        repository.setActivationId(datasource.get("activationId").toString());
86

    
87
        repository.setDescription(datasource.get("description").toString());
88
        if (repository.getDescription().equals("null"))
89
            repository.setDescription("");
90

    
91
        repository.setIssn(datasource.get("issn").toString());
92
        repository.setLissn(datasource.get("lissn").toString());
93
        if (repository.getLissn().equals("null"))
94
            repository.setLissn("");
95
        repository.setEissn(datasource.get("eissn").toString());
96
        if (repository.getEissn().equals("null"))
97
            repository.setEissn("");
98
        repository.setRegisteredBy(datasource.get("registeredby").toString());
99

    
100
        /* managed field */
101
        repository.setRegistered(Boolean.parseBoolean(datasource.get("managed").toString()));
102

    
103
        //subjects
104

    
105
        repository.setAggregator(datasource.get("aggregator").toString());
106
        repository.setCollectedFrom(datasource.get("collectedfrom").toString());
107

    
108
        //TODO change organization to list
109
        JSONArray organizations = ((JSONArray) datasource.get("organizations"));
110
        if (organizations.length() != 0) {
111
            repository.setOrganization(((JSONArray) datasource.get("organizations")).getJSONObject(0).get("legalname").toString());
112
            String countryCode = ((JSONArray) datasource.get("organizations")).getJSONObject(0).get("country").toString();
113
            repository.setCountryCode(countryCode);
114
        }
115

    
116
        repository.setConsentTermsOfUse(convertStringToBoolean(datasource.get("consentTermsOfUse").toString()));
117
        repository.setConsentTermsOfUseDate(null);
118
        repository.setLastConsentTermsOfUseDate(null);
119
        try {
120
            repository.setConsentTermsOfUseDate(convertStringToDate(datasource.get("consentTermsOfUseDate").toString()));
121
            repository.setLastConsentTermsOfUseDate(convertStringToDate(datasource.get("lastConsentTermsOfUseDate").toString()));
122
        } catch (JSONException e) {
123
            LOGGER.info("Error setting consentTermsOfUseDate date and lastConsentTermsOfUseDate", e);
124
        }
125
        repository.setFullTextDownload(convertStringToBoolean(datasource.get("fullTextDownload").toString()));
126

    
127
        /* identities field  */
128

    
129
        return repository;
130
    }
131

    
132
    public static Boolean convertStringToBoolean(String value) {
133
        return value.equals("null") ? null : Boolean.valueOf(value);
134
    }
135

    
136
    public static Date convertStringToDate(String date) {
137

    
138
        if (Objects.equals(date, "null"))
139
            return null;
140

    
141
        SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
142
        try {
143
            return formatter.parse(date);
144
        } catch (ParseException e) {
145
            LOGGER.error(e);
146
        }
147
        return null;
148
    }
149

    
150
    public static String convertDateToString(Date date) {
151

    
152
        if (Objects.equals(date, null))
153
            return null;
154

    
155
        SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
156
        return formatter.format(date);
157
    }
158

    
159
    public static Double toDouble(String number) {
160
        if (Objects.equals(number, "null"))
161
            return 0.0;
162
        else
163
            return Double.valueOf(number);
164
    }
165

    
166
    public static List<RepositorySnippet> jsonToRepositorySnippetList(JSONObject json) throws JSONException {
167

    
168
        List<RepositorySnippet> resultSet = new ArrayList<>();
169
        JSONArray rs = json.getJSONArray("datasourceInfo");
170
        for (int i = 0; i < rs.length(); i++)
171
            resultSet.add(jsonToRepositorySnippetObject(rs.getJSONObject(i)));
172
        return resultSet;
173
    }
174

    
175
    public static RepositorySnippet jsonToRepositorySnippetObject(JSONObject repositorySnippetObject) throws JSONException {
176

    
177

    
178
        RepositorySnippet repositorySnippet = new RepositorySnippet();
179

    
180
//        JSONObject datasource = repositorySnippetObject.getJSONObject("datasource");
181

    
182

    
183
        repositorySnippet.setId(repositorySnippetObject.get("id").toString());
184
        repositorySnippet.setOfficialname(repositorySnippetObject.get("officialname").toString());
185

    
186
        repositorySnippet.setEnglishname(repositorySnippetObject.get("englishname").toString());
187
        if (repositorySnippet.getEnglishname().equals("null"))
188
            repositorySnippet.setEnglishname("");
189

    
190
        repositorySnippet.setWebsiteurl(repositorySnippetObject.get("websiteurl").toString());
191
        if (repositorySnippet.getWebsiteurl().equals("null"))
192
            repositorySnippet.setWebsiteurl("");
193

    
194
        repositorySnippet.setRegisteredby(repositorySnippetObject.get("registeredby").toString());
195
        if (repositorySnippet.getRegisteredby().equals("null"))
196
            repositorySnippet.setRegisteredby("");
197

    
198
        repositorySnippet.setConsentTermsOfUse(repositorySnippetObject.get("consenttermsofuse").toString());
199
        repositorySnippet.setFullTextDownload(repositorySnippetObject.get("fulltextdownload").toString());
200
        repositorySnippet.setConsentTermsOfUseDate(convertStringToDate(repositorySnippetObject.get("consenttermsofusedate").toString()));
201

    
202
        return repositorySnippet;
203

    
204
    }
205

    
206
    public static List<Repository> jsonToRepositoryList(JSONObject json) throws JSONException {
207

    
208
        List<Repository> resultSet = new ArrayList<>();
209
        JSONArray rs = json.getJSONArray("datasourceInfo");
210
        for (int i = 0; i < rs.length(); i++)
211
            resultSet.add(jsonToRepositoryObject(rs.getJSONObject(i)));
212
        return resultSet;
213
    }
214

    
215
    public static List<RepositoryInterface> jsonToRepositoryInterfaceList(JSONObject json) throws JSONException {
216

    
217
        List<RepositoryInterface> resultSet = new ArrayList<>();
218
        JSONArray rs = json.getJSONArray("api");
219
        for (int i = 0; i < rs.length(); i++)
220
            resultSet.add(jsonToRepositoryInterfaceObject(rs.getJSONObject(i)));
221
        return resultSet;
222
    }
223

    
224
    public static RepositoryInterface jsonToRepositoryInterfaceObject(JSONObject repositoryInterfaceObject) throws JSONException {
225

    
226
        RepositoryInterface repositoryInterface = new RepositoryInterface();
227

    
228
        repositoryInterface.setId(repositoryInterfaceObject.get("id").toString());
229
        repositoryInterface.setAccessProtocol(repositoryInterfaceObject.get("protocol").toString());
230
        repositoryInterface.setContentDescription(repositoryInterfaceObject.get("contentdescription").toString());
231

    
232
        //        /* 07-04-2022 | "typology" -> "eoscDatasourceType" */
233
//        // TODO: enable in future release
234
//        try { // FIXME: remove attemp to get typology if eoscDatasourceType fails
235
//            repositoryInterface.setTypology(repositoryInterfaceObject.get("eoscDatasourceType").toString());
236
//        } catch (JSONException e) {
237
//            repositoryInterface.setTypology(repositoryInterfaceObject.get("typology").toString());
238
//        }
239
        repositoryInterface.setTypology(repositoryInterfaceObject.get("typology").toString());
240
//        <--
241

    
242
        repositoryInterface.setCompliance(repositoryInterfaceObject.get("compatibility").toString());
243
        repositoryInterface.setLastCollectionDate(repositoryInterfaceObject.get("lastCollectionDate").toString());
244

    
245
        repositoryInterface.setBaseUrl(repositoryInterfaceObject.get("baseurl").toString());
246
        repositoryInterface.setRemovable(Boolean.parseBoolean(repositoryInterfaceObject.get("removable").toString()));
247

    
248

    
249
        // repositoryInterface.setMetadataIdentifierPath(repositoryInterfaceObject.get("metadataIdentifierPath").toString());
250
        repositoryInterface.setDesiredCompatibilityLevel(repositoryInterfaceObject.get("compatibility").toString());
251
        //repositoryInterface.setActive(Boolean.parseBoolean(repositoryInterfaceObject.get("active").toString()));
252

    
253

    
254
        Map<String, String> accessParams = new HashMap<>();
255
        Map<String, String> extraFields = new HashMap<>();
256

    
257
        ObjectMapper mapper = new ObjectMapper();
258
        JSONArray apiparams = repositoryInterfaceObject.getJSONArray("apiParams");
259

    
260
        for (int i = 0; i < apiparams.length(); i++)
261
            accessParams.put(apiparams.getJSONObject(i).getString("param"), apiparams.getJSONObject(i).getString("value"));
262

    
263
        repositoryInterface.setAccessParams(accessParams);
264

    
265
        return repositoryInterface;
266
    }
267

    
268
    public static String repositoryObjectToJson(Repository repository) throws JSONException, JsonProcessingException {
269

    
270
        HashMap<String, Object> repositoryMap = new HashMap<>();
271
        ObjectMapper mapper = new ObjectMapper();
272

    
273
        repositoryMap.put("id", repository.getId());
274
        repositoryMap.put("openaireId", getOpenaireId(repository.getId()));
275
        repositoryMap.put("officialname", repository.getOfficialName());
276
        repositoryMap.put("englishname", repository.getEnglishName());
277
        repositoryMap.put("websiteurl", repository.getWebsiteUrl());
278
        repositoryMap.put("logourl", repository.getLogoUrl());
279
        repositoryMap.put("contactemail", repository.getContactEmail());
280
        repositoryMap.put("longitude", repository.getLongitude().toString());
281
        repositoryMap.put("latitude", repository.getLatitude().toString());
282
        repositoryMap.put("timezone", repository.getTimezone());
283

    
284
        repositoryMap.put("namespaceprefix", repository.getNamespacePrefix() != null ? repository.getNamespacePrefix() : "");
285
        repositoryMap.put("languages", repository.getOdLanguages() != null ? repository.getOdLanguages() : "");
286

    
287
        repositoryMap.put("dateofcollection", repository.getDateOfCollection() != null ? convertDateToString(repository.getDateOfCollection()) : "");
288

    
289
        /*
290
         * typology -> platform
291
         * datasource class -> typology
292
         * */
293
//        repositoryMap.put("eoscDatasourceType", repository.getDatasourceClass()); // TODO: enable in future release
294
        repositoryMap.put("platform", repository.getTypology());
295

    
296
        repositoryMap.put("dateofvalidation", repository.getDateOfCollection() != null ? convertDateToString(repository.getDateOfCollection()) : "");
297
        repositoryMap.put("activationId", repository.getActivationId() != null ? repository.getActivationId() : "");
298

    
299
        repositoryMap.put("description", repository.getDescription());
300

    
301
        repositoryMap.put("eissn", repository.getEissn() != null ? repository.getEissn() : "");
302
        repositoryMap.put("issn", repository.getIssn() != null ? repository.getIssn() : "");
303
        repositoryMap.put("lissn", repository.getLissn() != null ? repository.getLissn() : "");
304

    
305
        repositoryMap.put("registeredby", repository.getRegisteredBy());
306

    
307
        repositoryMap.put("aggregator", repository.getAggregator() != null ? repository.getAggregator() : "");
308
        repositoryMap.put("collectedfrom", repository.getCollectedFrom() != null ? repository.getCollectedFrom() : "");
309

    
310
        repositoryMap.put("managed", repository.isRegistered());
311

    
312
        Map<String, String> organization = new HashMap<>();
313
        organization.put("legalname", repository.getOrganization());
314
        organization.put("country", repository.getCountryCode());
315
        organization.put("legalshortname", "");
316
        organization.put("websiteurl", "");
317
        organization.put("logourl", "");
318

    
319
        List organizations = new ArrayList();
320
        organizations.add(organization);
321
        repositoryMap.put("organizations", organizations);
322

    
323
        //TODO check identitites
324
        //Map<String,String> identity = new HashMap<>();
325

    
326
        if (repository.getPiwikInfo() != null) {
327
            Map<String, Object> identity = new HashMap<>();
328
            HashSet<Map<String, Object>> identities = new HashSet<>();
329

    
330
            identity.put("issuertype", "piwik");
331
            identity.put("pid", "piwik:" + repository.getPiwikInfo().getSiteId());
332

    
333
            identities.add(identity);
334

    
335
            repositoryMap.put("identities", identities);
336
        }
337

    
338
        repositoryMap.put("subjects", "");
339
        repositoryMap.put("consentTermsOfUse", repository.getConsentTermsOfUse());
340
        repositoryMap.put("fullTextDownload", repository.getFullTextDownload());
341
        repositoryMap.put("consentTermsOfUseDate", convertDateToString(repository.getConsentTermsOfUseDate()));
342
        repositoryMap.put("lastConsentTermsOfUseDate", convertDateToString(repository.getLastConsentTermsOfUseDate()));
343

    
344
        return mapper.writeValueAsString(repositoryMap);
345
    }
346

    
347
    public static String repositoryInterfaceObjectToJson(Repository repository, RepositoryInterface repositoryInterface) throws JSONException {
348

    
349
        JSONObject jsonObject = new JSONObject();
350

    
351
        jsonObject.put("id", repositoryInterface.getId());
352
        jsonObject.put("protocol", repositoryInterface.getAccessProtocol());
353
        jsonObject.put("datasource", repository.getId());
354
        jsonObject.put("contentdescription", repositoryInterface.getContentDescription());
355
        jsonObject.put("typology", repositoryInterface.getTypology());
356
        jsonObject.put("compatibility", repositoryInterface.getDesiredCompatibilityLevel());
357
        jsonObject.put("compatibilityOverride", repositoryInterface.getDesiredCompatibilityLevel());
358

    
359
        jsonObject.put("lastCollectionTotal", "");
360

    
361
        jsonObject.put("lastCollectionDate", repositoryInterface.getLastCollectionDate());
362
        jsonObject.put("lastAggregationTotal", "");
363
        jsonObject.put("lastAggregationDate", "");
364
        jsonObject.put("lastDownloadTotal", "");
365
        jsonObject.put("lastDownloadDate", "");
366

    
367
        jsonObject.put("baseurl", repositoryInterface.getBaseUrl());
368
        jsonObject.put("removable", repositoryInterface.isRemovable());
369

    
370

    
371
        JSONArray apiparams = new JSONArray();
372
        for (String param : repositoryInterface.getAccessParams().keySet()) {
373
            JSONObject jo = new JSONObject();
374
            jo.put("param", param);
375
            jo.put("value", repositoryInterface.getAccessParams().get(param));
376
            apiparams.put(jo);
377
        }
378
        jsonObject.put("apiParams", apiparams);
379

    
380

    
381
//        jsonObject.put("metadataIdentifierPath",repositoryInterface.getMetadataIdentifierPath());
382

    
383

    
384
        return jsonObject.toString();
385
    }
386

    
387
    public static ArrayList<String> readFile(String filename) {
388
        String line;
389
        ArrayList<String> list = new ArrayList<String>();
390
        try {
391
            //InputStream in = Converter.class.getResourceAsStream("resources/eu/dnetlib/repo/manager/service/utils/"+filename);
392
            InputStream in = Converter.class.getClass().getResourceAsStream("/eu/**/" + filename);
393
            BufferedReader br = new BufferedReader(new InputStreamReader(in));
394
            while ((line = br.readLine()) != null) {
395
                list.add(line.trim());
396
            }
397
            br.close();
398
        } catch (IOException e) {
399
            LOGGER.debug("Error opening file!");
400
            LOGGER.error(e);
401
        }
402
        return list;
403
    }
404

    
405
    public static List<AggregationDetails> getAggregationHistoryFromJson(JSONArray aggregationInfo) throws JSONException {
406
        List<AggregationDetails> aggregationDetailsList = new ArrayList<>();
407

    
408
        for (int i = 0; i < aggregationInfo.length(); i++)
409
            aggregationDetailsList.add(jsonToAggregationDetails(aggregationInfo.getJSONObject(i)));
410

    
411
        return aggregationDetailsList;
412
    }
413

    
414
    private static AggregationDetails jsonToAggregationDetails(JSONObject aggregationObject) throws JSONException {
415

    
416
        AggregationDetails aggregationDetails = new AggregationDetails();
417

    
418
        if (aggregationObject.has("collectionMode"))
419
            aggregationDetails.setCollectionMode(aggregationObject.get("collectionMode").toString());
420
        if (aggregationObject.has("indexedVersion"))
421
            aggregationDetails.setIndexedVersion(Boolean.parseBoolean(aggregationObject.get("indexedVersion").toString()));
422

    
423
        aggregationDetails.setAggregationStage(aggregationObject.get("aggregationStage").toString());
424
        aggregationDetails.setDate(convertStringToDate(aggregationObject.get("date").toString()));
425
        aggregationDetails.setNumberOfRecords(Integer.parseInt(aggregationObject.get("numberOfRecords").toString()));
426

    
427
        return aggregationDetails;
428
    }
429

    
430
    public static List<Timezone> toTimezones(List<String> timezones) {
431

    
432
        List<Timezone> tmz = new ArrayList<>();
433
        for (String t : timezones) {
434
            String[] s = t.split("\t");
435
            tmz.add(new Timezone(s[1], Double.parseDouble(s[0])));
436
        }
437
        return tmz;
438
    }
439

    
440
    private static String getOpenaireId(String repositoryId) {
441
        if (repositoryId != null && repositoryId.contains("::"))
442
            return repositoryId.split("::")[0] + "::" + DigestUtils.md5Hex(repositoryId.split("::")[1]);
443
        return null;
444
    }
445

    
446
}
(1-1/6)