Project

General

Profile

1
package eu.dnetlib.repo.manager.utils;
2

    
3
import com.fasterxml.jackson.core.JsonProcessingException;
4
import com.fasterxml.jackson.databind.ObjectMapper;
5
import eu.dnetlib.domain.data.Repository;
6
import eu.dnetlib.domain.data.RepositoryInterface;
7
import eu.dnetlib.repo.manager.domain.AggregationDetails;
8
import eu.dnetlib.repo.manager.domain.RepositorySnippet;
9
import eu.dnetlib.repo.manager.domain.Timezone;
10
import org.apache.commons.codec.digest.DigestUtils;
11
import org.apache.log4j.Logger;
12
import org.json.JSONArray;
13
import org.json.JSONException;
14
import org.json.JSONObject;
15
import org.springframework.stereotype.Component;
16

    
17
import java.io.BufferedReader;
18
import java.io.IOException;
19
import java.io.InputStream;
20
import java.io.InputStreamReader;
21
import java.text.ParseException;
22
import java.text.SimpleDateFormat;
23
import java.util.*;
24

    
25
@Component
26
public class Converter {
27

    
28
    private static final Logger LOGGER = Logger.getLogger(Converter.class);
29

    
30
    public static Repository jsonToRepositoryObject(JSONObject repositoryObject) throws JSONException {
31

    
32
        Repository repository = new Repository();
33

    
34
//        JSONObject datasource = repositoryObject.getJSONObject("datasource");
35
        JSONObject datasource = repositoryObject;
36

    
37
        //if( datasource.equals(null))
38
        //    return null;
39

    
40
        repository.setId(datasource.get("id").toString());
41
        repository.setOfficialName(datasource.get("officialname").toString());
42

    
43
        repository.setEnglishName(datasource.get("englishname").toString());
44
        if (repository.getEnglishName().equals("null"))
45
            repository.setEnglishName("");
46

    
47
        repository.setWebsiteUrl(datasource.get("websiteurl").toString());
48
        if (repository.getWebsiteUrl().equals("null"))
49
            repository.setWebsiteUrl("");
50

    
51
        repository.setLogoUrl(datasource.get("logourl").toString());
52
        if (repository.getLogoUrl().equals("null"))
53
            repository.setLogoUrl("");
54

    
55
        repository.setContactEmail(datasource.get("contactemail").toString());
56
        if (repository.getContactEmail().equals("null"))
57
            repository.setContactEmail("");
58

    
59

    
60
        repository.setLatitude(toDouble(datasource.get("latitude").toString()));
61
        repository.setLongitude(toDouble(datasource.get("longitude").toString()));
62
        Double timezone = toDouble(datasource.get("timezone").toString());
63
        repository.setTimezone(timezone != null ? timezone : 0.0);
64
        repository.setNamespacePrefix(datasource.get("namespaceprefix").toString());
65
        repository.setOdLanguages(datasource.get("languages").toString());
66
        repository.setDateOfValidation(convertStringToDate(datasource.get("dateofvalidation").toString()));
67

    
68
        /*  typology -> platform
69
         *  datasource class -> typology */
70
        repository.setTypology(datasource.get("platform").toString());
71
        if (repository.getTypology().equals("null"))
72
            repository.setTypology("");
73
        repository.setDatasourceClass(datasource.get("typology").toString());
74

    
75
        repository.setDateOfCollection(convertStringToDate(datasource.get("dateofcollection").toString()));
76
        repository.setActivationId(datasource.get("activationId").toString());
77

    
78
        repository.setDescription(datasource.get("description").toString());
79
        if (repository.getDescription().equals("null"))
80
            repository.setDescription("");
81

    
82
        repository.setIssn(datasource.get("issn").toString());
83
        repository.setLissn(datasource.get("lissn").toString());
84
        if (repository.getLissn().equals("null"))
85
            repository.setLissn("");
86
        repository.setEissn(datasource.get("eissn").toString());
87
        if (repository.getEissn().equals("null"))
88
            repository.setEissn("");
89
        repository.setRegisteredBy(datasource.get("registeredby").toString());
90

    
91
        /* managed field */
92
        repository.setRegistered(Boolean.parseBoolean(datasource.get("managed").toString()));
93

    
94
        //subjects
95

    
96
        repository.setAggregator(datasource.get("aggregator").toString());
97
        repository.setCollectedFrom(datasource.get("collectedfrom").toString());
98

    
99
        //TODO change organization to list
100
        JSONArray organizations = ((JSONArray) datasource.get("organizations"));
101
        if (organizations.length() != 0) {
102
            repository.setOrganization(((JSONArray) datasource.get("organizations")).getJSONObject(0).get("legalname").toString());
103
            String countryCode = ((JSONArray) datasource.get("organizations")).getJSONObject(0).get("country").toString();
104
            repository.setCountryCode(countryCode);
105
        }
106

    
107
        repository.setConsentTermsOfUse(convertStringToBoolean(datasource.get("consentTermsOfUse").toString()));
108
        try {
109
            repository.setConsentTermsOfUseDate(convertStringToDate(datasource.get("consentTermsOfUseDate").toString()));
110
        } catch (JSONException e) {
111
            repository.setConsentTermsOfUseDate(null);
112
        }
113
        repository.setFullTextDownload(convertStringToBoolean(datasource.get("fullTextDownload").toString()));
114

    
115
        /* identities field  */
116

    
117
        return repository;
118
    }
119

    
120
    public static Boolean convertStringToBoolean(String value) {
121
        return value.equals("null") ? null : Boolean.valueOf(value);
122
    }
123

    
124
    public static Date convertStringToDate(String date) {
125

    
126
        if (Objects.equals(date, "null"))
127
            return null;
128

    
129
        SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
130
        try {
131
            return formatter.parse(date);
132
        } catch (ParseException e) {
133
            LOGGER.error(e);
134
        }
135
        return null;
136
    }
137

    
138
    public static String convertDateToString(Date date) {
139

    
140
        if (Objects.equals(date, null))
141
            return null;
142

    
143
        SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
144
        return formatter.format(date);
145
    }
146

    
147
    public static Double toDouble(String number) {
148
        if (Objects.equals(number, "null"))
149
            return 0.0;
150
        else
151
            return Double.valueOf(number);
152
    }
153

    
154
    public static List<RepositorySnippet> jsonToRepositorySnippetList(JSONObject json) throws JSONException {
155

    
156
        List<RepositorySnippet> resultSet = new ArrayList<>();
157
        JSONArray rs = json.getJSONArray("datasourceInfo");
158
        for (int i = 0; i < rs.length(); i++)
159
            resultSet.add(jsonToRepositorySnippetObject(rs.getJSONObject(i)));
160
        return resultSet;
161
    }
162

    
163
    public static RepositorySnippet jsonToRepositorySnippetObject(JSONObject repositorySnippetObject) throws JSONException {
164

    
165

    
166
        RepositorySnippet repositorySnippet = new RepositorySnippet();
167

    
168
//        JSONObject datasource = repositorySnippetObject.getJSONObject("datasource");
169

    
170

    
171
        repositorySnippet.setId(repositorySnippetObject.get("id").toString());
172
        repositorySnippet.setOfficialname(repositorySnippetObject.get("officialname").toString());
173

    
174
        repositorySnippet.setEnglishname(repositorySnippetObject.get("englishname").toString());
175
        if (repositorySnippet.getEnglishname().equals("null"))
176
            repositorySnippet.setEnglishname("");
177

    
178
        repositorySnippet.setWebsiteurl(repositorySnippetObject.get("websiteurl").toString());
179
        if (repositorySnippet.getWebsiteurl().equals("null"))
180
            repositorySnippet.setWebsiteurl("");
181

    
182
        repositorySnippet.setRegisteredby(repositorySnippetObject.get("registeredby").toString());
183
        if (repositorySnippet.getRegisteredby().equals("null"))
184
            repositorySnippet.setRegisteredby("");
185

    
186
        repositorySnippet.setConsentTermsOfUse(repositorySnippetObject.get("consenttermsofuse").toString());
187
        repositorySnippet.setFullTextDownload(repositorySnippetObject.get("fulltextdownload").toString());
188
        repositorySnippet.setConsentTermsOfUseDate(convertStringToDate(repositorySnippetObject.get("consenttermsofusedate").toString()));
189

    
190
        return repositorySnippet;
191

    
192
    }
193

    
194
    public static List<Repository> jsonToRepositoryList(JSONObject json) throws JSONException {
195

    
196
        List<Repository> resultSet = new ArrayList<>();
197
        JSONArray rs = json.getJSONArray("datasourceInfo");
198
        for (int i = 0; i < rs.length(); i++)
199
            resultSet.add(jsonToRepositoryObject(rs.getJSONObject(i)));
200
        return resultSet;
201
    }
202

    
203
    public static List<RepositoryInterface> jsonToRepositoryInterfaceList(JSONObject json) throws JSONException {
204

    
205
        List<RepositoryInterface> resultSet = new ArrayList<>();
206
        JSONArray rs = json.getJSONArray("api");
207
        for (int i = 0; i < rs.length(); i++)
208
            resultSet.add(jsonToRepositoryInterfaceObject(rs.getJSONObject(i)));
209
        return resultSet;
210
    }
211

    
212
    public static RepositoryInterface jsonToRepositoryInterfaceObject(JSONObject repositoryInterfaceObject) throws JSONException {
213

    
214
        RepositoryInterface repositoryInterface = new RepositoryInterface();
215

    
216
        repositoryInterface.setId(repositoryInterfaceObject.get("id").toString());
217
        repositoryInterface.setAccessProtocol(repositoryInterfaceObject.get("protocol").toString());
218
        repositoryInterface.setContentDescription(repositoryInterfaceObject.get("contentdescription").toString());
219
        repositoryInterface.setTypology(repositoryInterfaceObject.get("typology").toString());
220
        repositoryInterface.setCompliance(repositoryInterfaceObject.get("compatibility").toString());
221
        repositoryInterface.setLastCollectionDate(repositoryInterfaceObject.get("lastCollectionDate").toString());
222

    
223
        repositoryInterface.setBaseUrl(repositoryInterfaceObject.get("baseurl").toString());
224
        repositoryInterface.setRemovable(Boolean.parseBoolean(repositoryInterfaceObject.get("removable").toString()));
225

    
226

    
227
        // repositoryInterface.setMetadataIdentifierPath(repositoryInterfaceObject.get("metadataIdentifierPath").toString());
228
        repositoryInterface.setDesiredCompatibilityLevel(repositoryInterfaceObject.get("compatibility").toString());
229
        //repositoryInterface.setActive(Boolean.parseBoolean(repositoryInterfaceObject.get("active").toString()));
230

    
231

    
232
        Map<String, String> accessParams = new HashMap<>();
233
        Map<String, String> extraFields = new HashMap<>();
234

    
235
        ObjectMapper mapper = new ObjectMapper();
236
        JSONArray apiparams = repositoryInterfaceObject.getJSONArray("apiParams");
237

    
238
        for (int i = 0; i < apiparams.length(); i++)
239
            accessParams.put(apiparams.getJSONObject(i).getString("param"), apiparams.getJSONObject(i).getString("value"));
240

    
241
        repositoryInterface.setAccessParams(accessParams);
242

    
243
        return repositoryInterface;
244
    }
245

    
246
    public static String repositoryObjectToJson(Repository repository) throws JSONException, JsonProcessingException {
247

    
248
        HashMap<String, Object> repositoryMap = new HashMap<>();
249
        ObjectMapper mapper = new ObjectMapper();
250

    
251
        repositoryMap.put("id", repository.getId());
252
        repositoryMap.put("openaireId", getOpenaireId(repository.getId()));
253
        repositoryMap.put("officialname", repository.getOfficialName());
254
        repositoryMap.put("englishname", repository.getEnglishName());
255
        repositoryMap.put("websiteurl", repository.getWebsiteUrl());
256
        repositoryMap.put("logourl", repository.getLogoUrl());
257
        repositoryMap.put("contactemail", repository.getContactEmail());
258
        repositoryMap.put("longitude", repository.getLongitude().toString());
259
        repositoryMap.put("latitude", repository.getLatitude().toString());
260
        repositoryMap.put("timezone", repository.getTimezone());
261

    
262
        repositoryMap.put("namespaceprefix", repository.getNamespacePrefix() != null ? repository.getNamespacePrefix() : "");
263
        repositoryMap.put("languages", repository.getOdLanguages() != null ? repository.getOdLanguages() : "");
264

    
265
        repositoryMap.put("dateofcollection", repository.getDateOfCollection() != null ? convertDateToString(repository.getDateOfCollection()) : "");
266

    
267
        /*
268
         * typology -> platform
269
         * datasource class -> typology
270
         * */
271
        repositoryMap.put("typology", repository.getDatasourceClass());
272
        repositoryMap.put("platform", repository.getTypology());
273

    
274
        repositoryMap.put("dateofvalidation", repository.getDateOfCollection() != null ? convertDateToString(repository.getDateOfCollection()) : "");
275
        repositoryMap.put("activationId", repository.getActivationId() != null ? repository.getActivationId() : "");
276

    
277
        repositoryMap.put("description", repository.getDescription());
278

    
279
        repositoryMap.put("eissn", repository.getEissn() != null ? repository.getEissn() : "");
280
        repositoryMap.put("issn", repository.getIssn() != null ? repository.getIssn() : "");
281
        repositoryMap.put("lissn", repository.getLissn() != null ? repository.getLissn() : "");
282

    
283
        repositoryMap.put("registeredby", repository.getRegisteredBy());
284

    
285
        repositoryMap.put("aggregator", repository.getAggregator() != null ? repository.getAggregator() : "");
286
        repositoryMap.put("collectedfrom", repository.getCollectedFrom() != null ? repository.getCollectedFrom() : "");
287

    
288
        repositoryMap.put("managed", repository.isRegistered());
289

    
290
        Map<String, String> organization = new HashMap<>();
291
        organization.put("legalname", repository.getOrganization());
292
        organization.put("country", repository.getCountryCode());
293
        organization.put("legalshortname", "");
294
        organization.put("websiteurl", "");
295
        organization.put("logourl", "");
296

    
297
        List organizations = new ArrayList();
298
        organizations.add(organization);
299
        repositoryMap.put("organizations", organizations);
300

    
301
        //TODO check identitites
302
        //Map<String,String> identity = new HashMap<>();
303

    
304
        if (repository.getPiwikInfo() != null) {
305
            Map<String, Object> identity = new HashMap<>();
306
            HashSet<Map<String, Object>> identities = new HashSet<>();
307

    
308
            identity.put("issuertype", "piwik");
309
            identity.put("pid", "piwik:" + repository.getPiwikInfo().getSiteId());
310

    
311
            identities.add(identity);
312

    
313
            repositoryMap.put("identities", identities);
314
        }
315

    
316
        repositoryMap.put("subjects", "");
317
        repositoryMap.put("consentTermsOfUse", repository.getConsentTermsOfUse());
318
        repositoryMap.put("fullTextDownload", repository.getFullTextDownload());
319
        repositoryMap.put("consentTermsOfUseDate", convertDateToString(repository.getConsentTermsOfUseDate()));
320

    
321
        return mapper.writeValueAsString(repositoryMap);
322
    }
323

    
324
    public static String repositoryInterfaceObjectToJson(Repository repository, RepositoryInterface repositoryInterface) throws JSONException {
325

    
326
        JSONObject jsonObject = new JSONObject();
327

    
328
        jsonObject.put("id", repositoryInterface.getId());
329
        jsonObject.put("protocol", repositoryInterface.getAccessProtocol());
330
        jsonObject.put("datasource", repository.getId());
331
        jsonObject.put("contentdescription", repositoryInterface.getContentDescription());
332
        jsonObject.put("typology", repositoryInterface.getTypology());
333
        jsonObject.put("compatibility", repositoryInterface.getDesiredCompatibilityLevel());
334
        jsonObject.put("compatibilityOverride", repositoryInterface.getDesiredCompatibilityLevel());
335

    
336
        jsonObject.put("lastCollectionTotal", "");
337

    
338
        jsonObject.put("lastCollectionDate", repositoryInterface.getLastCollectionDate());
339
        jsonObject.put("lastAggregationTotal", "");
340
        jsonObject.put("lastAggregationDate", "");
341
        jsonObject.put("lastDownloadTotal", "");
342
        jsonObject.put("lastDownloadDate", "");
343

    
344
        jsonObject.put("baseurl", repositoryInterface.getBaseUrl());
345
        jsonObject.put("removable", repositoryInterface.isRemovable());
346

    
347

    
348
        JSONArray apiparams = new JSONArray();
349
        for (String param : repositoryInterface.getAccessParams().keySet()) {
350
            JSONObject jo = new JSONObject();
351
            jo.put("param", param);
352
            jo.put("value", repositoryInterface.getAccessParams().get(param));
353
            apiparams.put(jo);
354
        }
355
        jsonObject.put("apiParams", apiparams);
356

    
357

    
358
//        jsonObject.put("metadataIdentifierPath",repositoryInterface.getMetadataIdentifierPath());
359

    
360

    
361
        return jsonObject.toString();
362
    }
363

    
364
    public static ArrayList<String> readFile(String filename) {
365
        String line;
366
        ArrayList<String> list = new ArrayList<String>();
367
        try {
368
            //InputStream in = Converter.class.getResourceAsStream("resources/eu/dnetlib/repo/manager/service/utils/"+filename);
369
            InputStream in = Converter.class.getClass().getResourceAsStream("/eu/**/" + filename);
370
            BufferedReader br = new BufferedReader(new InputStreamReader(in));
371
            while ((line = br.readLine()) != null) {
372
                list.add(line.trim());
373
            }
374
            br.close();
375
        } catch (IOException e) {
376
            LOGGER.debug("Error opening file!");
377
            LOGGER.error(e);
378
        }
379
        return list;
380
    }
381

    
382
    public static List<AggregationDetails> getAggregationHistoryFromJson(JSONObject datasourceInfo) throws JSONException {
383
        JSONArray rs;
384
        List<AggregationDetails> aggregationDetailsList = new ArrayList<>();
385

    
386
        if (datasourceInfo.get("aggregationHistory") != null && !datasourceInfo.get("aggregationHistory").toString().equals("null")) {
387
            rs = new JSONArray(datasourceInfo.get("aggregationHistory").toString());
388

    
389
            for (int i = 0; i < rs.length(); i++)
390
                aggregationDetailsList.add(jsonToAggregationDetails(rs.getJSONObject(i)));
391
        }
392

    
393
        return aggregationDetailsList;
394
    }
395

    
396
    private static AggregationDetails jsonToAggregationDetails(JSONObject aggregationObject) throws JSONException {
397

    
398
        AggregationDetails aggregationDetails = new AggregationDetails();
399

    
400
        aggregationDetails.setAggregationStage(aggregationObject.get("aggregationStage").toString());
401
        if (aggregationObject.has("collectionMode"))
402
            aggregationDetails.setCollectionMode(aggregationObject.get("collectionMode").toString());
403
        if (aggregationObject.has("indexedVersion"))
404
            aggregationDetails.setIndexedVersion(Boolean.parseBoolean(aggregationObject.get("indexedVersion").toString()));
405
        aggregationDetails.setDate(convertStringToDate(aggregationObject.get("date").toString()));
406
        aggregationDetails.setNumberOfRecords(Integer.parseInt(aggregationObject.get("numberOfRecords").toString()));
407
        return aggregationDetails;
408
    }
409

    
410

    
411
    public static List<Timezone> toTimezones(List<String> timezones) {
412

    
413
        List<Timezone> tmz = new ArrayList<>();
414
        for (String t : timezones) {
415
            String[] s = t.split("\t");
416
            tmz.add(new Timezone(s[1], Double.parseDouble(s[0])));
417
        }
418
        return tmz;
419
    }
420

    
421
    private static String getOpenaireId(String repositoryId) {
422
        if (repositoryId != null && repositoryId.contains("::"))
423
            return repositoryId.split("::")[0] + "::" + DigestUtils.md5Hex(repositoryId.split("::")[1]);
424
        return null;
425
    }
426

    
427
}
(1-1/6)