Project

General

Profile

1
package eu.dnetlib.repo.manager.service.utils;
2

    
3
import com.fasterxml.jackson.core.JsonProcessingException;
4
import com.fasterxml.jackson.databind.ObjectMapper;
5
import eu.dnetlib.domain.data.Repository;
6
import eu.dnetlib.domain.data.RepositoryInterface;
7
import eu.dnetlib.repo.manager.service.domain.RepositorySnippet;
8
import eu.dnetlib.repo.manager.shared.AggregationDetails;
9
import eu.dnetlib.repo.manager.shared.Timezone;
10
import org.apache.commons.codec.digest.DigestUtils;
11
import org.apache.log4j.Logger;
12
import org.json.JSONArray;
13
import org.json.JSONException;
14
import org.json.JSONObject;
15

    
16
import java.io.BufferedReader;
17
import java.io.IOException;
18
import java.io.InputStream;
19
import java.io.InputStreamReader;
20
import java.text.ParseException;
21
import java.text.SimpleDateFormat;
22
import java.util.*;
23

    
24
public class Converter {
25

    
26
    private static final Logger LOGGER = Logger.getLogger(Converter.class);
27

    
28
    public static Repository jsonToRepositoryObject(JSONObject repositoryObject) throws JSONException {
29

    
30
        Repository repository = new Repository();
31
        
32
        JSONObject datasource = repositoryObject.getJSONObject("datasource");
33

    
34
        //if( datasource.equals(null))
35
        //    return null;
36

    
37
        repository.setId(datasource.get("id").toString());
38
        repository.setOfficialName(datasource.get("officialname").toString());
39

    
40
        repository.setEnglishName( datasource.get("englishname").toString());
41
        if(repository.getEnglishName().equals("null"))
42
            repository.setEnglishName("");
43

    
44
        repository.setWebsiteUrl(datasource.get("websiteurl").toString());
45
        if(repository.getWebsiteUrl().equals("null"))
46
            repository.setWebsiteUrl("");
47

    
48
        repository.setLogoUrl(datasource.get("logourl").toString());
49
        if(repository.getLogoUrl().equals("null"))
50
            repository.setLogoUrl("");
51

    
52
        repository.setContactEmail(datasource.get("contactemail").toString());
53
        if(repository.getContactEmail().equals("null"))
54
            repository.setContactEmail("");
55

    
56

    
57
        repository.setLatitude( toDouble(datasource.get("latitude").toString()));
58
        repository.setLongitude(toDouble(datasource.get("longitude").toString()));
59
        Double timezone = toDouble(datasource.get("timezone").toString());
60
        repository.setTimezone(timezone!=null?timezone:0.0);
61
        repository.setNamespacePrefix(datasource.get("namespaceprefix").toString());
62
        repository.setOdLanguages(datasource.get("languages").toString());
63
        repository.setDateOfValidation(convertStringToDate( datasource.get("dateofvalidation").toString()));
64

    
65
        /*  typology -> platform
66
         *  datasource class -> typology */
67
        repository.setTypology(datasource.get("platform").toString());
68
        if(repository.getTypology().equals("null"))
69
            repository.setTypology("");
70
        repository.setDatasourceClass(datasource.get("typology").toString());
71

    
72
        repository.setDateOfCollection(convertStringToDate( datasource.get("dateofcollection").toString()));
73
        repository.setActivationId(datasource.get("activationId").toString());
74

    
75
        repository.setDescription(datasource.get("description").toString());
76
        if(repository.getDescription().equals("null"))
77
            repository.setDescription("");
78

    
79
        repository.setIssn(datasource.get("issn").toString());
80
        repository.setLissn(datasource.get("lissn").toString());
81
        if(repository.getLissn().equals("null"))
82
            repository.setLissn("");
83
        repository.setEissn(datasource.get("eissn").toString());
84
        if(repository.getEissn().equals("null"))
85
            repository.setEissn("");
86
        repository.setRegisteredBy(datasource.get("registeredby").toString());
87

    
88
        /* managed field */
89
        repository.setRegistered(Boolean.parseBoolean(datasource.get("managed").toString()));
90

    
91
        //subjects
92

    
93
        repository.setAggregator(datasource.get("aggregator").toString());
94
        repository.setCollectedFrom(datasource.get("collectedfrom").toString());
95

    
96
        //TODO change organization to list
97
        JSONArray organizations = ((JSONArray)datasource.get("organizations"));
98
        if(organizations.length() != 0) {
99
            repository.setOrganization(((JSONArray) datasource.get("organizations")).getJSONObject(0).get("legalname").toString());
100
            String countryCode = ((JSONArray) datasource.get("organizations")).getJSONObject(0).get("country").toString();
101
            repository.setCountryCode(countryCode);
102
        }
103

    
104
        /* identities field  */
105

    
106
        return repository;
107
    }
108

    
109
    public static Date convertStringToDate(String date){
110

    
111
        if(Objects.equals(date, "null"))
112
            return null;
113

    
114
        SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
115
        try {
116
            return formatter.parse(date);
117
        } catch (ParseException e) {
118
            e.printStackTrace();
119
        }
120
        return null;
121
    }
122

    
123
    public static String convertDateToString(Date date){
124

    
125
        if(Objects.equals(date, null))
126
            return null;
127

    
128
        SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
129
        return formatter.format(date);
130
    }
131

    
132
    public static Double toDouble(String number){
133
        if(Objects.equals(number, "null"))
134
            return 0.0;
135
        else
136
            return Double.valueOf(number);
137
    }
138

    
139
    public static List<RepositorySnippet> jsonToRepositorySnippetList(JSONObject json) throws JSONException {
140

    
141
        List<RepositorySnippet> resultSet = new ArrayList<>();
142
        JSONArray rs = json.getJSONArray("datasourceInfo");
143
        for(int i=0;i<rs.length();i++)
144
            resultSet.add(jsonToRepositorySnippetObject( rs.getJSONObject(i)) );
145
        return resultSet;
146
    }
147

    
148
    private static RepositorySnippet jsonToRepositorySnippetObject(JSONObject repositorySnippetObject) throws JSONException {
149

    
150

    
151
        RepositorySnippet repositorySnippet = new RepositorySnippet();
152

    
153
//        JSONObject datasource = repositorySnippetObject.getJSONObject("datasource");
154

    
155

    
156
        repositorySnippet.setId(repositorySnippetObject.get("id").toString());
157
        repositorySnippet.setOfficialname(repositorySnippetObject.get("officialname").toString());
158

    
159
        repositorySnippet.setEnglishname( repositorySnippetObject.get("englishname").toString());
160
        if(repositorySnippet.getEnglishname().equals("null"))
161
            repositorySnippet.setEnglishname("");
162

    
163
        repositorySnippet.setWebsiteurl(repositorySnippetObject.get("websiteurl").toString());
164
        if(repositorySnippet.getWebsiteurl().equals("null"))
165
            repositorySnippet.setWebsiteurl("");
166

    
167
        repositorySnippet.setRegisteredby(repositorySnippetObject.get("registeredby").toString());
168
        if(repositorySnippet.getRegisteredby().equals("null"))
169
            repositorySnippet.setRegisteredby("");
170
        return repositorySnippet;
171

    
172
    }
173

    
174
    public static List<Repository> jsonToRepositoryList(JSONObject json) throws JSONException {
175

    
176
        List<Repository> resultSet = new ArrayList<>();
177
        JSONArray rs = json.getJSONArray("datasourceInfo");
178
        for(int i=0;i<rs.length();i++)
179
            resultSet.add(jsonToRepositoryObject( rs.getJSONObject(i)) );
180
        return resultSet;
181
    }
182

    
183
    public static List<RepositoryInterface> jsonToRepositoryInterfaceList(JSONObject json) throws JSONException {
184

    
185
        List<RepositoryInterface> resultSet = new ArrayList<>();
186
        JSONArray rs = json.getJSONArray("api");
187
        for(int i=0;i<rs.length();i++)
188
            resultSet.add(jsonToRepositoryInterfaceObject( rs.getJSONObject(i)) );
189
        return resultSet;
190
    }
191

    
192
    public static RepositoryInterface jsonToRepositoryInterfaceObject(JSONObject repositoryInterfaceObject) throws JSONException {
193

    
194
        RepositoryInterface repositoryInterface = new RepositoryInterface();
195

    
196
        repositoryInterface.setId(repositoryInterfaceObject.get("id").toString());
197
        repositoryInterface.setAccessProtocol(repositoryInterfaceObject.get("protocol").toString());
198
        repositoryInterface.setContentDescription(repositoryInterfaceObject.get("contentdescription").toString());
199
        repositoryInterface.setTypology(repositoryInterfaceObject.get("typology").toString());
200
        repositoryInterface.setCompliance(repositoryInterfaceObject.get("compatibility").toString());
201
        repositoryInterface.setLastCollectionDate(repositoryInterfaceObject.get("lastCollectionDate").toString());
202

    
203
        repositoryInterface.setBaseUrl(repositoryInterfaceObject.get("baseurl").toString());
204
        repositoryInterface.setRemovable(Boolean.parseBoolean(repositoryInterfaceObject.get("removable").toString()));
205

    
206

    
207
       // repositoryInterface.setMetadataIdentifierPath(repositoryInterfaceObject.get("metadataIdentifierPath").toString());
208
        repositoryInterface.setDesiredCompatibilityLevel(repositoryInterfaceObject.get("compatibility").toString());
209
        //repositoryInterface.setActive(Boolean.parseBoolean(repositoryInterfaceObject.get("active").toString()));
210

    
211

    
212
        Map<String, String> accessParams = new HashMap<>();
213
        Map<String, String> extraFields = new HashMap<>();
214

    
215
        ObjectMapper mapper = new ObjectMapper();
216
        JSONArray apiparams = repositoryInterfaceObject.getJSONArray("apiParams");
217

    
218
        for(int i=0;i<apiparams.length();i++)
219
            accessParams.put(apiparams.getJSONObject(i).getString("param"),apiparams.getJSONObject(i).getString("value"));
220

    
221
        repositoryInterface.setAccessParams(accessParams);
222

    
223
        return repositoryInterface;
224
    }
225

    
226
    public static String repositoryObjectToJson(Repository repository) throws JSONException, JsonProcessingException {
227

    
228
        JSONObject jsonObject = new JSONObject();
229

    
230
        jsonObject.put("id",repository.getId());
231
        jsonObject.put("openaireId",getOpenaireId(repository.getId()));
232
        jsonObject.put("officialname",repository.getOfficialName());
233
        jsonObject.put("englishname",repository.getEnglishName());
234
        jsonObject.put("websiteurl",repository.getWebsiteUrl());
235
        jsonObject.put("logourl",repository.getLogoUrl());
236
        jsonObject.put("contactemail",repository.getContactEmail());
237
        jsonObject.put("longitude",repository.getLongitude().toString());
238
        jsonObject.put("latitude",repository.getLatitude().toString());
239
        jsonObject.put("timezone",repository.getTimezone());
240

    
241
        jsonObject.put("namespaceprefix",repository.getNamespacePrefix()!=null?repository.getNamespacePrefix():"");
242
        jsonObject.put("languages",repository.getOdLanguages()!=null?repository.getOdLanguages():"");
243

    
244
        jsonObject.put("dateofcollection",repository.getDateOfCollection()!=null?convertDateToString(repository.getDateOfCollection()):"");
245

    
246
        /*
247
        * typology -> platform
248
        * datasource class -> typology
249
        * */
250
        jsonObject.put("typology",repository.getDatasourceClass());
251
        jsonObject.put("platform",repository.getTypology());
252

    
253
        jsonObject.put("dateofvalidation",repository.getDateOfCollection()!=null?convertDateToString(repository.getDateOfCollection()):"");
254
        jsonObject.put("activationId",repository.getActivationId()!=null?repository.getActivationId():"");
255

    
256
        jsonObject.put("description",repository.getDescription());
257

    
258
        jsonObject.put("eissn",repository.getEissn()!=null?repository.getEissn():"");
259
        jsonObject.put("issn",repository.getIssn()!=null?repository.getIssn():"");
260
        jsonObject.put("lissn",repository.getLissn()!=null?repository.getLissn():"");
261

    
262
        jsonObject.put("registeredby",repository.getRegisteredBy());
263

    
264
        jsonObject.put("aggregator",repository.getAggregator()!=null?repository.getAggregator():"");
265
        jsonObject.put("collectedfrom",repository.getCollectedFrom()!=null?repository.getCollectedFrom():"");
266

    
267
        jsonObject.put("managed",repository.isRegistered());
268

    
269
        JSONObject organization = new JSONObject();
270
        organization.put("legalname",repository.getOrganization());
271
        organization.put("country",repository.getCountryCode());
272
        organization.put("legalshortname","");
273
        organization.put("websiteurl","");
274
        organization.put("logourl","");
275

    
276
        JSONArray organizations = new JSONArray();
277
        organizations.put(organization);
278
        jsonObject.put("organizations",organizations);
279

    
280
        //TODO check identitites
281
        JSONArray identities = new JSONArray();
282
        identities.put(identities);
283
        jsonObject.put("identities",identities);
284

    
285

    
286

    
287
        //TODO check fields
288
       /* jsonObject.put("certificates",repository.getCertificates());
289
        jsonObject.put("citationguidelineurl",repository.getCitationGuidelineUrl());
290
        jsonObject.put("databaseaccessrestriction",repository.getDatabaseAccessRestriction());
291
        jsonObject.put("databaseaccesstype",repository.getDatabaseAccessType());
292
        jsonObject.put("datauploadrestriction",repository.getDataUploadRestriction());
293
        jsonObject.put("datauploadtype",repository.getDataUploadType());
294
        jsonObject.put("missionstatementurl",repository.getMissionStatementUrl());
295
        jsonObject.put("od_contenttypes",repository.getOdContentTypes());
296
        jsonObject.put("officialname",repository.getOfficialname());
297
        jsonObject.put("pidsystems",repository.getPidSystems());
298
        jsonObject.put("provenanceaction",repository.getProvenanceActionClass());
299
        jsonObject.put("qualitymanagementkind",repository.getQualityManagementKind());
300
        jsonObject.put("releaseenddate",convertDateToString(repository.getReleaseEndDate()));
301
        jsonObject.put("releasestartdate",convertDateToString(repository.getReleaseStartDate()));
302
        jsonObject.put("serviceprovider",repository.getServiceProvider());
303
        jsonObject.put("versioning",repository.getVersioning());
304
        //datasource.get("platform");
305
        //datasource.get("subjects");*/
306

    
307
        ObjectMapper mapper = new ObjectMapper();
308
        return mapper.writeValueAsString(jsonObject);
309
    }
310

    
311
    public static String repositoryInterfaceObjectToJson(Repository repository,RepositoryInterface repositoryInterface) throws JSONException {
312

    
313
        JSONObject jsonObject = new JSONObject();
314

    
315
        jsonObject.put("id",repositoryInterface.getId());
316
        jsonObject.put("protocol",repositoryInterface.getAccessProtocol());
317
        jsonObject.put("datasource",repository.getId());
318
        jsonObject.put("contentdescription",repositoryInterface.getContentDescription());
319
        jsonObject.put("typology",repositoryInterface.getTypology());
320
        jsonObject.put("compatibility",repositoryInterface.getDesiredCompatibilityLevel());
321
        jsonObject.put("compatibilityOverride",repositoryInterface.getDesiredCompatibilityLevel());
322

    
323
        jsonObject.put("lastCollectionTotal","");
324

    
325
        jsonObject.put("lastCollectionDate",repositoryInterface.getLastCollectionDate());
326
        jsonObject.put("lastAggregationTotal","");
327
        jsonObject.put("lastAggregationDate","");
328
        jsonObject.put("lastDownloadTotal","");
329
        jsonObject.put("lastDownloadDate","");
330

    
331
        jsonObject.put("baseurl",repositoryInterface.getBaseUrl());
332
        jsonObject.put("removable",repositoryInterface.isRemovable());
333

    
334
        
335
        JSONArray apiparams = new JSONArray();
336
        for(String param: repositoryInterface.getAccessParams().keySet()){
337
            JSONObject jo = new JSONObject();
338
            jo.put("param",param);
339
            jo.put("value",repositoryInterface.getAccessParams().get(param));
340
            apiparams.put(jo);
341
        }
342
        jsonObject.put("apiParams",apiparams);
343

    
344

    
345
//        jsonObject.put("metadataIdentifierPath",repositoryInterface.getMetadataIdentifierPath());
346

    
347

    
348
        return jsonObject.toString();
349
    }
350

    
351
    public static ArrayList<String> readFile(String filename) {
352
        String line;
353
        ArrayList<String> list = new ArrayList<String>();
354
        try {
355
            //InputStream in = Converter.class.getResourceAsStream("resources/eu/dnetlib/repo/manager/service/utils/"+filename);
356
            InputStream in = Converter.class.getClass().getResourceAsStream("/eu/**/" + filename);
357
            BufferedReader br = new BufferedReader(new InputStreamReader(in));
358
            while((line = br.readLine()) != null) {
359
                list.add(line.trim());
360
            }
361
            br.close();
362
        } catch (IOException e) {
363
            LOGGER.debug("Error opening file!");
364
            e.printStackTrace();
365
        }
366
        return list;
367
    }
368

    
369
    public static List<AggregationDetails> getAggregationHistoryFromJson(JSONObject datasourceInfo) throws JSONException {
370

    
371
        JSONArray rs = new JSONArray(datasourceInfo.get("aggregationHistory").toString());
372
        List<AggregationDetails> aggregationDetailsList = new ArrayList<>();
373
        for(int i=0;i<rs.length();i++)
374
            aggregationDetailsList.add(jsonToAggregationDetails(rs.getJSONObject(i)));
375
        return aggregationDetailsList;
376
    }
377

    
378
    private static AggregationDetails jsonToAggregationDetails(JSONObject aggregationObject) throws JSONException {
379

    
380
        AggregationDetails aggregationDetails = new AggregationDetails();
381

    
382
        aggregationDetails.setAggregationStage(aggregationObject.get("aggregationStage").toString());
383
        if(aggregationObject.has("collectionMode"))
384
            aggregationDetails.setCollectionMode(aggregationObject.get("collectionMode").toString());
385
        if(aggregationObject.has("indexedVersion"))
386
            aggregationDetails.setIndexedVersion(Boolean.parseBoolean(aggregationObject.get("indexedVersion").toString()));
387
        aggregationDetails.setDate(convertStringToDate(aggregationObject.get("date").toString()));
388
        aggregationDetails.setNumberOfRecords(Integer.parseInt(aggregationObject.get("numberOfRecords").toString()));
389
        return aggregationDetails;
390
    }
391

    
392
    public static AggregationDetails getLastCollectionFromJson(JSONObject datasourceInfo) throws JSONException {
393

    
394
        if( datasourceInfo.get("lastCollection").equals(null))
395
            return null;
396

    
397
        return jsonToAggregationDetails(datasourceInfo.getJSONObject("lastCollection"));
398
    }
399

    
400
    public static AggregationDetails getLastTransformationFromJson(JSONObject datasourceInfo) throws JSONException {
401

    
402
        if( datasourceInfo.get("lastTransformation").equals(null))
403
            return null;
404

    
405
        return jsonToAggregationDetails(datasourceInfo.getJSONObject("lastTransformation"));
406
    }
407

    
408
    public static List<Timezone> toTimezones(List<String> timezones) {
409

    
410
        List<Timezone> tmz = new ArrayList<>();
411
        for(String t : timezones){
412
            String[] s = t.split("\t");
413
            tmz.add(new Timezone(s[1],Double.parseDouble(s[0])));
414
        }
415
        return tmz;
416
    }
417

    
418
    public static String getOpenaireId(String repositoryId) {
419
        if (repositoryId != null && repositoryId.contains("::"))
420
            return repositoryId.split("::")[0] + "::" + DigestUtils.md5Hex(repositoryId.split("::")[1]);
421
        return null;
422
    }
423

    
424
}
(1-1/2)