Project

General

Profile

1
package eu.dnetlib.repo.manager.utils;
2

    
3
import com.fasterxml.jackson.core.JsonProcessingException;
4
import com.fasterxml.jackson.databind.ObjectMapper;
5
import eu.dnetlib.domain.data.Repository;
6
import eu.dnetlib.domain.data.RepositoryInterface;
7
import eu.dnetlib.repo.manager.domain.AggregationDetails;
8
import eu.dnetlib.repo.manager.domain.RepositorySnippet;
9
import eu.dnetlib.repo.manager.domain.Timezone;
10
import org.apache.commons.codec.digest.DigestUtils;
11
import org.apache.log4j.Logger;
12
import org.json.JSONArray;
13
import org.json.JSONException;
14
import org.json.JSONObject;
15
import org.springframework.stereotype.Component;
16

    
17
import java.io.BufferedReader;
18
import java.io.IOException;
19
import java.io.InputStream;
20
import java.io.InputStreamReader;
21
import java.text.ParseException;
22
import java.text.SimpleDateFormat;
23
import java.util.*;
24

    
25
@Component
26
public class Converter {
27

    
28
    private static final Logger LOGGER = Logger.getLogger(Converter.class);
29

    
30
    public Repository toRepository(JSONObject repositoryObject) throws JSONException {
31

    
32
        Repository repository = new Repository();
33

    
34
//        JSONObject datasource = repositoryObject.getJSONObject("datasource");
35
        JSONObject datasource = repositoryObject;
36

    
37
        //if( datasource.equals(null))
38
        //    return null;
39

    
40
        repository.setId(datasource.get("id").toString());
41
        repository.setOfficialName(datasource.get("officialname").toString());
42

    
43
        repository.setEnglishName(datasource.get("englishname").toString());
44
        if (repository.getEnglishName().equals("null"))
45
            repository.setEnglishName("");
46

    
47
        repository.setWebsiteUrl(datasource.get("websiteurl").toString());
48
        if (repository.getWebsiteUrl().equals("null"))
49
            repository.setWebsiteUrl("");
50

    
51
        repository.setLogoUrl(datasource.get("logourl").toString());
52
        if (repository.getLogoUrl().equals("null"))
53
            repository.setLogoUrl("");
54

    
55
        repository.setContactEmail(datasource.get("contactemail").toString());
56
        if (repository.getContactEmail().equals("null"))
57
            repository.setContactEmail("");
58

    
59

    
60
        repository.setLatitude(toDouble(datasource.get("latitude").toString()));
61
        repository.setLongitude(toDouble(datasource.get("longitude").toString()));
62
        Double timezone = toDouble(datasource.get("timezone").toString());
63
        repository.setTimezone(timezone != null ? timezone : 0.0);
64
        repository.setNamespacePrefix(datasource.get("namespaceprefix").toString());
65
        repository.setOdLanguages(datasource.get("languages").toString());
66
        repository.setDateOfValidation(toDate(datasource.get("dateofvalidation").toString()));
67

    
68
        /*  typology -> platform
69
         *  datasource class -> typology */
70
        repository.setTypology(datasource.get("platform").toString());
71
        if (repository.getTypology().equals("null"))
72
            repository.setTypology("");
73

    
74
//        // TODO: enable in future release
75
//        /* 07-04-2022 | "typology" -> "eoscDatasourceType" */
76
//        try { // FIXME: remove attemp to get typology if eoscDatasourceType fails
77
//            repository.setDatasourceClass(datasource.get("eoscDatasourceType").toString());
78
//        } catch (JSONException e) {
79
//            repository.setDatasourceClass(datasource.get("typology").toString());
80
//        }
81
        repository.setDatasourceClass(datasource.get("typology").toString());
82
//        <--
83

    
84
        repository.setDateOfCollection(toDate(datasource.get("dateofcollection").toString()));
85
        repository.setActivationId(datasource.get("activationId").toString());
86

    
87
        repository.setDescription(datasource.get("description").toString());
88
        if (repository.getDescription().equals("null"))
89
            repository.setDescription("");
90

    
91
        repository.setIssn(datasource.get("issn").toString());
92
        repository.setLissn(datasource.get("lissn").toString());
93
        if (repository.getLissn().equals("null"))
94
            repository.setLissn("");
95
        repository.setEissn(datasource.get("eissn").toString());
96
        if (repository.getEissn().equals("null"))
97
            repository.setEissn("");
98
        repository.setRegisteredBy(datasource.get("registeredby").toString());
99

    
100
        /* managed field */
101
        repository.setRegistered(Boolean.parseBoolean(datasource.get("managed").toString()));
102

    
103
        //subjects
104

    
105
        repository.setAggregator(datasource.get("aggregator").toString());
106
        repository.setCollectedFrom(datasource.get("collectedfrom").toString());
107

    
108
        //TODO change organization to list
109
        JSONArray organizations = ((JSONArray) datasource.get("organizations"));
110
        if (organizations.length() != 0) {
111
            repository.setOrganization(((JSONArray) datasource.get("organizations")).getJSONObject(0).get("legalname").toString());
112
            String countryCode = ((JSONArray) datasource.get("organizations")).getJSONObject(0).get("country").toString();
113
            repository.setCountryCode(countryCode);
114
        }
115

    
116
        repository.setConsentTermsOfUse(convertStringToBoolean(datasource.get("consentTermsOfUse").toString()));
117
        repository.setConsentTermsOfUseDate(null);
118
        repository.setLastConsentTermsOfUseDate(null);
119
        try {
120
            repository.setConsentTermsOfUseDate(toDate(datasource.get("consentTermsOfUseDate").toString()));
121
            repository.setLastConsentTermsOfUseDate(toDate(datasource.get("lastConsentTermsOfUseDate").toString()));
122
        } catch (JSONException e) {
123
            LOGGER.error("Error setting consentTermsOfUseDate date and lastConsentTermsOfUseDate", e);
124
        }
125
        repository.setFullTextDownload(convertStringToBoolean(datasource.get("fullTextDownload").toString()));
126

    
127
        /* identities field  */
128

    
129
        return repository;
130
    }
131

    
132
    public RepositorySnippet toRepositorySnippet(JSONObject repositorySnippetObject) throws JSONException {
133

    
134

    
135
        RepositorySnippet repositorySnippet = new RepositorySnippet();
136

    
137
//        JSONObject datasource = repositorySnippetObject.getJSONObject("datasource");
138

    
139

    
140
        repositorySnippet.setId(repositorySnippetObject.get("id").toString());
141
        repositorySnippet.setOfficialname(repositorySnippetObject.get("officialname").toString());
142

    
143
        repositorySnippet.setEnglishname(repositorySnippetObject.get("englishname").toString());
144
        if (repositorySnippet.getEnglishname().equals("null"))
145
            repositorySnippet.setEnglishname("");
146

    
147
        repositorySnippet.setWebsiteurl(repositorySnippetObject.get("websiteurl").toString());
148
        if (repositorySnippet.getWebsiteurl().equals("null"))
149
            repositorySnippet.setWebsiteurl("");
150

    
151
        repositorySnippet.setRegisteredby(repositorySnippetObject.get("registeredby").toString());
152
        if (repositorySnippet.getRegisteredby().equals("null"))
153
            repositorySnippet.setRegisteredby("");
154

    
155
        repositorySnippet.setConsentTermsOfUse(repositorySnippetObject.get("consenttermsofuse").toString());
156
        repositorySnippet.setFullTextDownload(repositorySnippetObject.get("fulltextdownload").toString());
157
        repositorySnippet.setConsentTermsOfUseDate(toDate(repositorySnippetObject.get("consenttermsofusedate").toString()));
158

    
159
        return repositorySnippet;
160

    
161
    }
162

    
163
    public List<Repository> toRepositoryList(JSONObject json) throws JSONException {
164

    
165
        List<Repository> resultSet = new ArrayList<>();
166
        JSONArray rs = json.getJSONArray("datasourceInfo");
167
        for (int i = 0; i < rs.length(); i++)
168
            resultSet.add(toRepository(rs.getJSONObject(i)));
169
        return resultSet;
170
    }
171

    
172
    public List<RepositoryInterface> toRepositoryInterfaceList(JSONObject json) throws JSONException {
173

    
174
        List<RepositoryInterface> resultSet = new ArrayList<>();
175
        JSONArray rs = json.getJSONArray("api");
176
        for (int i = 0; i < rs.length(); i++)
177
            resultSet.add(toRepositoryInterface(rs.getJSONObject(i)));
178
        return resultSet;
179
    }
180

    
181
    public RepositoryInterface toRepositoryInterface(JSONObject repositoryInterfaceObject) throws JSONException {
182

    
183
        RepositoryInterface repositoryInterface = new RepositoryInterface();
184

    
185
        repositoryInterface.setId(repositoryInterfaceObject.get("id").toString());
186
        repositoryInterface.setAccessProtocol(repositoryInterfaceObject.get("protocol").toString());
187
        repositoryInterface.setContentDescription(repositoryInterfaceObject.get("contentdescription").toString());
188

    
189
        //        /* 07-04-2022 | "typology" -> "eoscDatasourceType" */
190
//        // TODO: enable in future release
191
//        try { // FIXME: remove attemp to get typology if eoscDatasourceType fails
192
//            repositoryInterface.setTypology(repositoryInterfaceObject.get("eoscDatasourceType").toString());
193
//        } catch (JSONException e) {
194
//            repositoryInterface.setTypology(repositoryInterfaceObject.get("typology").toString());
195
//        }
196
        repositoryInterface.setTypology(repositoryInterfaceObject.get("typology").toString());
197
//        <--
198

    
199
        repositoryInterface.setCompliance(repositoryInterfaceObject.get("compatibility").toString());
200
        repositoryInterface.setLastCollectionDate(repositoryInterfaceObject.get("lastCollectionDate").toString());
201

    
202
        repositoryInterface.setBaseUrl(repositoryInterfaceObject.get("baseurl").toString());
203
        repositoryInterface.setRemovable(Boolean.parseBoolean(repositoryInterfaceObject.get("removable").toString()));
204

    
205

    
206
        // repositoryInterface.setMetadataIdentifierPath(repositoryInterfaceObject.get("metadataIdentifierPath").toString());
207
        repositoryInterface.setDesiredCompatibilityLevel(repositoryInterfaceObject.get("compatibility").toString());
208
        //repositoryInterface.setActive(Boolean.parseBoolean(repositoryInterfaceObject.get("active").toString()));
209

    
210

    
211
        Map<String, String> accessParams = new HashMap<>();
212
        Map<String, String> extraFields = new HashMap<>();
213

    
214
        ObjectMapper mapper = new ObjectMapper();
215
        JSONArray apiparams = repositoryInterfaceObject.getJSONArray("apiParams");
216

    
217
        for (int i = 0; i < apiparams.length(); i++)
218
            accessParams.put(apiparams.getJSONObject(i).getString("param"), apiparams.getJSONObject(i).getString("value"));
219

    
220
        repositoryInterface.setAccessParams(accessParams);
221

    
222
        return repositoryInterface;
223
    }
224

    
225
    public String toJson(Repository repository) throws JsonProcessingException {
226

    
227
        HashMap<String, Object> repositoryMap = new HashMap<>();
228
        ObjectMapper mapper = new ObjectMapper();
229

    
230
        repositoryMap.put("id", repository.getId());
231
        repositoryMap.put("openaireId", getOpenaireId(repository.getId()));
232
        repositoryMap.put("officialname", repository.getOfficialName());
233
        repositoryMap.put("englishname", repository.getEnglishName());
234
        repositoryMap.put("websiteurl", repository.getWebsiteUrl());
235
        repositoryMap.put("logourl", repository.getLogoUrl());
236
        repositoryMap.put("contactemail", repository.getContactEmail());
237
        repositoryMap.put("longitude", repository.getLongitude().toString());
238
        repositoryMap.put("latitude", repository.getLatitude().toString());
239
        repositoryMap.put("timezone", repository.getTimezone());
240

    
241
        repositoryMap.put("namespaceprefix", repository.getNamespacePrefix() != null ? repository.getNamespacePrefix() : "");
242
        repositoryMap.put("languages", repository.getOdLanguages() != null ? repository.getOdLanguages() : "");
243

    
244
        repositoryMap.put("dateofcollection", repository.getDateOfCollection() != null ? toString(repository.getDateOfCollection()) : "");
245

    
246
        /*
247
         * typology -> platform
248
         * datasource class -> typology
249
         * */
250
//        repositoryMap.put("eoscDatasourceType", repository.getDatasourceClass()); // TODO: enable in future release
251
        repositoryMap.put("platform", repository.getTypology());
252

    
253
        repositoryMap.put("dateofvalidation", repository.getDateOfCollection() != null ? toString(repository.getDateOfCollection()) : "");
254
        repositoryMap.put("activationId", repository.getActivationId() != null ? repository.getActivationId() : "");
255

    
256
        repositoryMap.put("description", repository.getDescription());
257

    
258
        repositoryMap.put("eissn", repository.getEissn() != null ? repository.getEissn() : "");
259
        repositoryMap.put("issn", repository.getIssn() != null ? repository.getIssn() : "");
260
        repositoryMap.put("lissn", repository.getLissn() != null ? repository.getLissn() : "");
261

    
262
        repositoryMap.put("registeredby", repository.getRegisteredBy());
263

    
264
        repositoryMap.put("aggregator", repository.getAggregator() != null ? repository.getAggregator() : "");
265
        repositoryMap.put("collectedfrom", repository.getCollectedFrom() != null ? repository.getCollectedFrom() : "");
266

    
267
        repositoryMap.put("managed", repository.isRegistered());
268

    
269
        Map<String, String> organization = new HashMap<>();
270
        organization.put("legalname", repository.getOrganization());
271
        organization.put("country", repository.getCountryCode());
272
        organization.put("legalshortname", "");
273
        organization.put("websiteurl", "");
274
        organization.put("logourl", "");
275

    
276
        List organizations = new ArrayList();
277
        organizations.add(organization);
278
        repositoryMap.put("organizations", organizations);
279

    
280
        //TODO check identitites
281
        //Map<String,String> identity = new HashMap<>();
282

    
283
        if (repository.getPiwikInfo() != null) {
284
            Map<String, Object> identity = new HashMap<>();
285
            HashSet<Map<String, Object>> identities = new HashSet<>();
286

    
287
            identity.put("issuertype", "piwik");
288
            identity.put("pid", "piwik:" + repository.getPiwikInfo().getSiteId());
289

    
290
            identities.add(identity);
291

    
292
            repositoryMap.put("identities", identities);
293
        }
294

    
295
        repositoryMap.put("subjects", "");
296
        repositoryMap.put("consentTermsOfUse", repository.getConsentTermsOfUse());
297
        repositoryMap.put("fullTextDownload", repository.getFullTextDownload());
298
        repositoryMap.put("consentTermsOfUseDate", toString(repository.getConsentTermsOfUseDate()));
299
        repositoryMap.put("lastConsentTermsOfUseDate", toString(repository.getLastConsentTermsOfUseDate()));
300

    
301
        return mapper.writeValueAsString(repositoryMap);
302
    }
303

    
304
    public String toJson(Repository repository, RepositoryInterface repositoryInterface) throws JSONException {
305

    
306
        JSONObject jsonObject = new JSONObject();
307

    
308
        jsonObject.put("id", repositoryInterface.getId());
309
        jsonObject.put("protocol", repositoryInterface.getAccessProtocol());
310
        jsonObject.put("datasource", repository.getId());
311
        jsonObject.put("contentdescription", repositoryInterface.getContentDescription());
312
        jsonObject.put("typology", repositoryInterface.getTypology());
313
        jsonObject.put("compatibility", repositoryInterface.getDesiredCompatibilityLevel());
314
        jsonObject.put("compatibilityOverride", repositoryInterface.getDesiredCompatibilityLevel());
315

    
316
        jsonObject.put("lastCollectionTotal", "");
317

    
318
        jsonObject.put("lastCollectionDate", repositoryInterface.getLastCollectionDate());
319
        jsonObject.put("lastAggregationTotal", "");
320
        jsonObject.put("lastAggregationDate", "");
321
        jsonObject.put("lastDownloadTotal", "");
322
        jsonObject.put("lastDownloadDate", "");
323

    
324
        jsonObject.put("baseurl", repositoryInterface.getBaseUrl());
325
        jsonObject.put("removable", repositoryInterface.isRemovable());
326

    
327

    
328
        JSONArray apiparams = new JSONArray();
329
        for (String param : repositoryInterface.getAccessParams().keySet()) {
330
            JSONObject jo = new JSONObject();
331
            jo.put("param", param);
332
            jo.put("value", repositoryInterface.getAccessParams().get(param));
333
            apiparams.put(jo);
334
        }
335
        jsonObject.put("apiParams", apiparams);
336

    
337

    
338
//        jsonObject.put("metadataIdentifierPath",repositoryInterface.getMetadataIdentifierPath());
339

    
340

    
341
        return jsonObject.toString();
342
    }
343

    
344
    public ArrayList<String> readFile(String filename) {
345
        String line;
346
        ArrayList<String> list = new ArrayList<String>();
347
        try {
348
            //InputStream in = Converter.class.getResourceAsStream("resources/eu/dnetlib/repo/manager/service/utils/"+filename);
349
            InputStream in = Converter.class.getClass().getResourceAsStream("/eu/**/" + filename);
350
            BufferedReader br = new BufferedReader(new InputStreamReader(in));
351
            while ((line = br.readLine()) != null) {
352
                list.add(line.trim());
353
            }
354
            br.close();
355
        } catch (IOException e) {
356
            LOGGER.debug("Error opening file!");
357
            LOGGER.error(e);
358
        }
359
        return list;
360
    }
361

    
362
    public List<AggregationDetails> toAggregationHistory(JSONArray aggregationInfo) throws JSONException {
363
        List<AggregationDetails> aggregationDetailsList = new ArrayList<>();
364

    
365
        for (int i = 0; i < aggregationInfo.length(); i++)
366
            aggregationDetailsList.add(toAggregationDetails(aggregationInfo.getJSONObject(i)));
367

    
368
        return aggregationDetailsList;
369
    }
370

    
371
    public List<Timezone> toTimezones(List<String> timezones) {
372

    
373
        List<Timezone> tmz = new ArrayList<>();
374
        for (String t : timezones) {
375
            String[] s = t.split("\t");
376
            tmz.add(new Timezone(s[1], Double.parseDouble(s[0])));
377
        }
378
        return tmz;
379
    }
380

    
381
    private String getOpenaireId(String repositoryId) {
382
        if (repositoryId != null && repositoryId.contains("::"))
383
            return repositoryId.split("::")[0] + "::" + DigestUtils.md5Hex(repositoryId.split("::")[1]);
384
        return null;
385
    }
386

    
387
    private Boolean convertStringToBoolean(String value) {
388
        return value.equals("null") ? null : Boolean.valueOf(value);
389
    }
390

    
391
    private Date toDate(String date) {
392

    
393
        if (Objects.equals(date, "null"))
394
            return null;
395

    
396
        SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
397
        try {
398
            return formatter.parse(date);
399
        } catch (ParseException e) {
400
            LOGGER.error(e);
401
        }
402
        return null;
403
    }
404

    
405
    private String toString(Date date) {
406

    
407
        if (Objects.equals(date, null))
408
            return null;
409

    
410
        SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
411
        return formatter.format(date);
412
    }
413

    
414
    private Double toDouble(String number) {
415
        if (Objects.equals(number, "null"))
416
            return 0.0;
417
        else
418
            return Double.valueOf(number);
419
    }
420

    
421
    private AggregationDetails toAggregationDetails(JSONObject aggregationObject) throws JSONException {
422

    
423
        AggregationDetails aggregationDetails = new AggregationDetails();
424

    
425
        if (aggregationObject.has("collectionMode"))
426
            aggregationDetails.setCollectionMode(aggregationObject.get("collectionMode").toString());
427
        if (aggregationObject.has("indexedVersion"))
428
            aggregationDetails.setIndexedVersion(Boolean.parseBoolean(aggregationObject.get("indexedVersion").toString()));
429

    
430
        aggregationDetails.setAggregationStage(aggregationObject.get("aggregationStage").toString());
431
        aggregationDetails.setDate(toDate(aggregationObject.get("date").toString()));
432
        aggregationDetails.setNumberOfRecords(Integer.parseInt(aggregationObject.get("numberOfRecords").toString()));
433

    
434
        return aggregationDetails;
435
    }
436
}
(1-1/6)