Project

General

Profile

1
package eu.dnetlib.data.mapreduce.hbase.dataexport;
2

    
3
import com.googlecode.protobuf.format.JsonFormat;
4
import eu.dnetlib.data.proto.*;
5
import eu.dnetlib.dhp.schema.oaf.*;
6
import eu.dnetlib.dhp.schema.util.ProtoUtils;
7

    
8
import java.io.Serializable;
9
import java.util.stream.Collectors;
10

    
11
public class ProtoConverter implements Serializable {
12

    
13
    public static Oaf convert(OafProtos.Oaf oaf) {
14
        try {
15
            if (oaf.getKind() == KindProtos.Kind.entity)
16
                return convertEntity(oaf);
17
            else {
18
                return convertRelation(oaf);
19
            }
20
        } catch (Throwable e) {
21
            throw new RuntimeException("error on getting " + JsonFormat.printToString(oaf), e);
22
        }
23
    }
24

    
25
    private static Relation convertRelation(OafProtos.Oaf oaf) {
26
        final OafProtos.OafRel r = oaf.getRel();
27
        final Relation rel = new Relation();
28
        rel.setDataInfo(mapDataInfo(oaf.getDataInfo()));
29
        rel.setLastupdatetimestamp(oaf.getLastupdatetimestamp());
30
        rel.setSource(r.getSource());
31
        rel.setTarget(r.getTarget());
32
        rel.setRelType(r.getRelType().toString());
33
        rel.setSubRelType(r.getSubRelType().toString());
34
        rel.setRelClass(r.getRelClass());
35
        rel.setCollectedFrom(r.getCollectedfromCount() > 0 ?
36
                r.getCollectedfromList().stream()
37
                        .map(kv -> mapKV(kv))
38
                        .collect(Collectors.toList()) : null);
39
        return rel;
40
    }
41

    
42
    private static OafEntity convertEntity(OafProtos.Oaf oaf) {
43

    
44
        switch (oaf.getEntity().getType()) {
45
            case result:
46
                return convertResult(oaf);
47
            case project:
48
                return convertProject(oaf);
49
            case datasource:
50
                return convertDataSource(oaf);
51
            case organization:
52
                return convertOrganization(oaf);
53
            default:
54
                throw new RuntimeException("received unknown type");
55
        }
56
    }
57

    
58
    private static Organization convertOrganization(OafProtos.Oaf oaf) {
59
        final OrganizationProtos.Organization.Metadata m = oaf.getEntity().getOrganization().getMetadata();
60
        final Organization org = setOaf(new Organization(), oaf);
61
        setEntity(org, oaf);
62
        org.setLegalshortname(mapStringField(m.getLegalshortname()));
63
        org.setLegalname(mapStringField(m.getLegalname()));
64
        org.setAlternativeNames(m.getAlternativeNamesList().
65
                stream()
66
                .map(ProtoConverter::mapStringField)
67
                .collect(Collectors.toList()));
68
        org.setWebsiteurl(mapStringField(m.getWebsiteurl()));
69
        org.setLogourl(mapStringField(m.getLogourl()));
70
        org.setEclegalbody(mapStringField(m.getEclegalbody()));
71
        org.setEclegalperson(mapStringField(m.getEclegalperson()));
72
        org.setEcnonprofit(mapStringField(m.getEcnonprofit()));
73
        org.setEcresearchorganization(mapStringField(m.getEcresearchorganization()));
74
        org.setEchighereducation(mapStringField(m.getEchighereducation()));
75
        org.setEcinternationalorganizationeurinterests(mapStringField(m.getEcinternationalorganizationeurinterests()));
76
        org.setEcinternationalorganization(mapStringField(m.getEcinternationalorganization()));
77
        org.setEcenterprise(mapStringField(m.getEcenterprise()));
78
        org.setEcsmevalidated(mapStringField(m.getEcsmevalidated()));
79
        org.setEcnutscode(mapStringField(m.getEcnutscode()));
80
        org.setCountry(mapQualifier(m.getCountry()));
81

    
82
        return org;
83
    }
84

    
85
    private static Datasource convertDataSource(OafProtos.Oaf oaf) {
86
        final DatasourceProtos.Datasource.Metadata m = oaf.getEntity().getDatasource().getMetadata();
87
        final Datasource datasource = setOaf(new Datasource(), oaf);
88
        setEntity(datasource, oaf);
89
        datasource.setAccessinfopackage(m.getAccessinfopackageList()
90
                .stream()
91
                .map(ProtoConverter::mapStringField)
92
                .collect(Collectors.toList()));
93
        datasource.setCertificates(mapStringField(m.getCertificates()));
94
        datasource.setCitationguidelineurl(mapStringField(m.getCitationguidelineurl()));
95
        datasource.setContactemail(mapStringField(m.getContactemail()));
96
        datasource.setDatabaseaccessrestriction(mapStringField(m.getDatabaseaccessrestriction()));
97
        datasource.setDatabaseaccesstype(mapStringField(m.getDatabaseaccesstype()));
98
        datasource.setDataprovider(mapBoolField(m.getDataprovider()));
99
        datasource.setDatasourcetype(mapQualifier(m.getDatasourcetype()));
100
        datasource.setDatauploadrestriction(mapStringField(m.getDatauploadrestriction()));
101
        datasource.setCitationguidelineurl(mapStringField(m.getCitationguidelineurl()));
102
        datasource.setDatauploadtype(mapStringField(m.getDatauploadtype()));
103
        datasource.setDateofvalidation(mapStringField(m.getDateofvalidation()));
104
        datasource.setDescription(mapStringField(m.getDescription()));
105
        datasource.setEnglishname(mapStringField(m.getEnglishname()));
106
        datasource.setLatitude(mapStringField(m.getLatitude()));
107
        datasource.setLongitude(mapStringField(m.getLongitude()));
108
        datasource.setLogourl(mapStringField(m.getLogourl()));
109
        datasource.setMissionstatementurl(mapStringField(m.getMissionstatementurl()));
110
        datasource.setNamespaceprefix(mapStringField(m.getNamespaceprefix()));
111
        datasource.setOdcontenttypes(m.getOdcontenttypesList()
112
                .stream()
113
                .map(ProtoConverter::mapStringField)
114
                .collect(Collectors.toList()));
115
        datasource.setOdlanguages(m.getOdlanguagesList()
116
                .stream()
117
                .map(ProtoConverter::mapStringField)
118
                .collect(Collectors.toList()));
119
        datasource.setOdnumberofitems(mapStringField(m.getOdnumberofitems()));
120
        datasource.setOdnumberofitemsdate(mapStringField(m.getOdnumberofitemsdate()));
121
        datasource.setOdpolicies(mapStringField(m.getOdpolicies()));
122
        datasource.setOfficialname(mapStringField(m.getOfficialname()));
123
        datasource.setOpenairecompatibility(mapQualifier(m.getOpenairecompatibility()));
124
        datasource.setPidsystems(mapStringField(m.getPidsystems()));
125
        datasource.setPolicies(m.getPoliciesList()
126
                .stream()
127
                .map(ProtoConverter::mapKV)
128
                .collect(Collectors.toList()));
129
        datasource.setQualitymanagementkind(mapStringField(m.getQualitymanagementkind()));
130
        datasource.setReleaseenddate(mapStringField(m.getReleaseenddate()));
131
        datasource.setServiceprovider(mapBoolField(m.getServiceprovider()));
132
        datasource.setReleasestartdate(mapStringField(m.getReleasestartdate()));
133
        datasource.setSubjects(m.getSubjectsList()
134
                .stream()
135
                .map(ProtoConverter::mapStructuredProperty)
136
                .collect(Collectors.toList()));
137
        datasource.setVersioning(mapBoolField(m.getVersioning()));
138
        datasource.setWebsiteurl(mapStringField(m.getWebsiteurl()));
139
        datasource.setJournal(mapJournal(m.getJournal()));
140

    
141

    
142
        return datasource;
143
    }
144

    
145
    private static Project convertProject(OafProtos.Oaf oaf) {
146
        final ProjectProtos.Project.Metadata m = oaf.getEntity().getProject().getMetadata();
147
        final Project project = setOaf(new Project(), oaf);
148
        setEntity(project, oaf);
149
        project.setAcronym(mapStringField(m.getAcronym()));
150
        project.setCallidentifier(mapStringField(m.getCallidentifier()));
151
        project.setCode(mapStringField(m.getCode()));
152
        project.setContactemail(mapStringField(m.getContactemail()));
153
        project.setContactfax(mapStringField(m.getContactfax()));
154
        project.setContactfullname(mapStringField(m.getContactfullname()));
155
        project.setContactphone(mapStringField(m.getContactphone()));
156
        project.setContracttype(mapQualifier(m.getContracttype()));
157
        project.setCurrency(mapStringField(m.getCurrency()));
158
        project.setDuration(mapStringField(m.getDuration()));
159
        project.setEcarticle29_3(mapStringField(m.getEcarticle293()));
160
        project.setEcsc39(mapStringField(m.getEcsc39()));
161
        project.setOamandatepublications(mapStringField(m.getOamandatepublications()));
162
        project.setStartdate(mapStringField(m.getStartdate()));
163
        project.setEnddate(mapStringField(m.getEnddate()));
164
        project.setFundedamount(m.getFundedamount());
165
        project.setTotalcost(m.getTotalcost());
166
        project.setKeywords(mapStringField(m.getKeywords()));
167
        project.setSubjects(m.getSubjectsList().stream()
168
                .map(sp -> mapStructuredProperty(sp))
169
                .collect(Collectors.toList()));
170
        project.setTitle(mapStringField(m.getTitle()));
171
        project.setWebsiteurl(mapStringField(m.getWebsiteurl()));
172
        project.setFundingtree(m.getFundingtreeList().stream()
173
                .map(f -> mapStringField(f))
174
                .collect(Collectors.toList()));
175
        project.setJsonextrainfo(mapStringField(m.getJsonextrainfo()));
176
        project.setSummary(mapStringField(m.getSummary()));
177
        project.setOptional1(mapStringField(m.getOptional1()));
178
        project.setOptional2(mapStringField(m.getOptional2()));
179
        return project;
180
    }
181

    
182
    private static Result convertResult(OafProtos.Oaf oaf) {
183
        switch (oaf.getEntity().getResult().getMetadata().getResulttype().getClassid()) {
184

    
185
            case "dataset":
186
                return createDataset(oaf);
187
            case "publication":
188
                return createPublication(oaf);
189
            case "software":
190
                return createSoftware(oaf);
191
            case "other":
192
                return createORP(oaf);
193
            default:
194
                throw new RuntimeException("received unknown type: " + oaf.getEntity().getResult().getMetadata().getResulttype().getClassid());
195
        }
196
    }
197

    
198
    private static Software createSoftware(OafProtos.Oaf oaf) {
199
        ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata();
200
        Software software = setOaf(new Software(), oaf);
201
        setEntity(software, oaf);
202
        setResult(software, oaf);
203

    
204
        software.setDocumentationUrl(m.getDocumentationUrlList()
205
                .stream()
206
                .map(ProtoConverter::mapStringField)
207
                .collect(Collectors.toList()));
208
        software.setLicense(m.getLicenseList()
209
                .stream()
210
                .map(ProtoConverter::mapStructuredProperty)
211
                .collect(Collectors.toList()));
212
        software.setCodeRepositoryUrl(ProtoUtils.mapStringField(m.getCodeRepositoryUrl()));
213
        software.setProgrammingLanguage(ProtoUtils.mapQualifier(m.getProgrammingLanguage()));
214
        return software;
215
    }
216

    
217
    private static OtherResearchProducts createORP(OafProtos.Oaf oaf) {
218
        ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata();
219
        OtherResearchProducts otherResearchProducts = setOaf(new OtherResearchProducts(), oaf);
220
        setEntity(otherResearchProducts, oaf);
221
        setResult(otherResearchProducts, oaf);
222
        otherResearchProducts.setContactperson(m.getContactpersonList()
223
                .stream()
224
                .map(ProtoConverter::mapStringField)
225
                .collect(Collectors.toList()));
226
        otherResearchProducts.setContactgroup(m.getContactgroupList()
227
                .stream()
228
                .map(ProtoConverter::mapStringField)
229
                .collect(Collectors.toList()));
230
        otherResearchProducts.setTool(m.getToolList()
231
                .stream()
232
                .map(ProtoConverter::mapStringField)
233
                .collect(Collectors.toList()));
234

    
235
        return otherResearchProducts;
236
    }
237

    
238
    private static Publication createPublication(OafProtos.Oaf oaf) {
239

    
240
        ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata();
241
        Publication publication = setOaf(new Publication(), oaf);
242
        setEntity(publication, oaf);
243
        setResult(publication, oaf);
244
        publication.setJournal(mapJournal(m.getJournal()));
245
        return publication;
246
    }
247

    
248
    private static Dataset createDataset(OafProtos.Oaf oaf) {
249

    
250
        ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata();
251
        Dataset dataset = setOaf(new Dataset(), oaf);
252
        setEntity(dataset, oaf);
253
        setResult(dataset, oaf);
254
        dataset.setStoragedate(ProtoUtils.mapStringField(m.getStoragedate()));
255
        dataset.setDevice(ProtoUtils.mapStringField(m.getDevice()));
256
        dataset.setSize(ProtoUtils.mapStringField(m.getSize()));
257
        dataset.setVersion(ProtoUtils.mapStringField(m.getVersion()));
258
        dataset.setLastmetadataupdate(ProtoUtils.mapStringField(m.getLastmetadataupdate()));
259
        dataset.setMetadataversionnumber(ProtoUtils.mapStringField(m.getMetadataversionnumber()));
260
        dataset.setGeolocation(m.getGeolocationList()
261
                .stream()
262
                .map(ProtoConverter::mapGeolocation)
263
                .collect(Collectors.toList()));
264
        return dataset;
265

    
266
    }
267

    
268
    public static <T extends Oaf> T setOaf(T oaf, OafProtos.Oaf o) {
269
        oaf.setDataInfo(mapDataInfo(o.getDataInfo()));
270
        oaf.setLastupdatetimestamp(o.getLastupdatetimestamp());
271
        return oaf;
272
    }
273

    
274
    public static <T extends OafEntity> T setEntity(T entity, OafProtos.Oaf oaf) {
275
        //setting Entity fields
276
        final OafProtos.OafEntity e = oaf.getEntity();
277
        entity.setId(e.getId());
278
        entity.setOriginalId(e.getOriginalIdList());
279
        entity.setCollectedfrom(e.getCollectedfromList()
280
                .stream()
281
                .map(ProtoConverter::mapKV)
282
                .collect(Collectors.toList()));
283
        entity.setPid(e.getPidList().stream()
284
                .map(ProtoConverter::mapStructuredProperty)
285
                .collect(Collectors.toList()));
286
        entity.setDateofcollection(entity.getDateofcollection());
287
        entity.setDateoftransformation(entity.getDateoftransformation());
288
        entity.setExtraInfo(e.getExtraInfoList()
289
                .stream()
290
                .map(ProtoConverter::mapExtraInfo)
291
                .collect(Collectors.toList()));
292
        return entity;
293
    }
294

    
295
    public static <T extends Result> T setResult(T entity, OafProtos.Oaf oaf) {
296
        //setting Entity fields
297
        final ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata();
298
        entity.setAuthor(m.getAuthorList()
299
                .stream()
300
                .map(ProtoConverter::mapAuthor)
301
                .collect(Collectors.toList()));
302
        entity.setResulttype(mapQualifier(m.getResulttype()));
303
        entity.setLanguage(ProtoUtils.mapQualifier(m.getLanguage()));
304
        entity.setCountry(m.getCountryList()
305
                .stream()
306
                .map(ProtoConverter::mapQualifier)
307
                .collect(Collectors.toList()));
308
        entity.setSubject(m.getSubjectList()
309
                .stream()
310
                .map(ProtoConverter::mapStructuredProperty)
311
                .collect(Collectors.toList()));
312
        entity.setTitle(m.getTitleList()
313
                .stream()
314
                .map(ProtoConverter::mapStructuredProperty)
315
                .collect(Collectors.toList()));
316
        entity.setRelevantdate(m.getRelevantdateList()
317
                .stream()
318
                .map(ProtoConverter::mapStructuredProperty)
319
                .collect(Collectors.toList()));
320
        entity.setDescription(m.getDescriptionList()
321
                .stream()
322
                .map(ProtoConverter::mapStringField)
323
                .collect(Collectors.toList()));
324
        entity.setDateofacceptance(ProtoUtils.mapStringField(m.getDateofacceptance()));
325
        entity.setPublisher(ProtoUtils.mapStringField(m.getPublisher()));
326
        entity.setEmbargoenddate(ProtoUtils.mapStringField(m.getEmbargoenddate()));
327
        entity.setSource(m.getSourceList()
328
                .stream()
329
                .map(ProtoConverter::mapStringField)
330
                .collect(Collectors.toList()));
331
        entity.setFulltext(m.getFulltextList()
332
                .stream()
333
                .map(ProtoConverter::mapStringField)
334
                .collect(Collectors.toList()));
335
        entity.setFormat(m.getFormatList()
336
                .stream()
337
                .map(ProtoConverter::mapStringField)
338
                .collect(Collectors.toList()));
339
        entity.setContributor(m.getContributorList()
340
                .stream()
341
                .map(ProtoConverter::mapStringField)
342
                .collect(Collectors.toList()));
343
        entity.setResourcetype(ProtoUtils.mapQualifier(m.getResourcetype()));
344
        entity.setCoverage(m.getCoverageList()
345
                .stream()
346
                .map(ProtoConverter::mapStringField)
347
                .collect(Collectors.toList()));
348
        entity.setRefereed(mapStringField(m.getRefereed()));
349
        entity.setContext(m.getContextList()
350
                .stream()
351
                .map(ProtoConverter::mapContext)
352
                .collect(Collectors.toList()));
353

    
354
        return entity;
355
    }
356

    
357
    private static Context mapContext(ResultProtos.Result.Context context) {
358

    
359
        final Context entity = new Context();
360
        entity.setId(context.getId());
361
        entity.setDataInfo(context.getDataInfoList()
362
                .stream()
363
                .map(ProtoConverter::mapDataInfo)
364
                .collect(Collectors.toList()));
365
        return entity;
366
    }
367

    
368

    
369
    public static KeyValue mapKV(FieldTypeProtos.KeyValue kv) {
370
        final KeyValue keyValue = new KeyValue();
371
        keyValue.setKey(kv.getKey());
372
        keyValue.setValue(kv.getValue());
373
        keyValue.setDataInfo(mapDataInfo(kv.getDataInfo()));
374
        return keyValue;
375
    }
376

    
377
    public static DataInfo mapDataInfo(FieldTypeProtos.DataInfo d) {
378
        final DataInfo dataInfo = new DataInfo();
379
        dataInfo.setDeletedbyinference(d.getDeletedbyinference());
380
        dataInfo.setInferenceprovenance(d.getInferenceprovenance());
381
        dataInfo.setInferred(d.getInferred());
382
        dataInfo.setInvisible(d.getInvisible());
383
        dataInfo.setProvenanceaction(mapQualifier(d.getProvenanceaction()));
384
        return dataInfo;
385
    }
386

    
387
    public static Qualifier mapQualifier(FieldTypeProtos.Qualifier q) {
388
        final Qualifier qualifier = new Qualifier();
389
        qualifier.setClassid(q.getClassid());
390
        qualifier.setClassname(q.getClassname());
391
        qualifier.setSchemeid(q.getSchemeid());
392
        qualifier.setSchemename(q.getSchemename());
393
        return qualifier;
394
    }
395

    
396
    public static StructuredProperty mapStructuredProperty(FieldTypeProtos.StructuredProperty sp) {
397
        final StructuredProperty structuredProperty = new StructuredProperty();
398
        structuredProperty.setValue(sp.getValue());
399
        structuredProperty.setQualifier(mapQualifier(sp.getQualifier()));
400
        structuredProperty.setDataInfo(mapDataInfo(sp.getDataInfo()));
401
        return structuredProperty;
402
    }
403

    
404
    public static ExtraInfo mapExtraInfo(FieldTypeProtos.ExtraInfo extraInfo) {
405
        final ExtraInfo entity = new ExtraInfo();
406
        entity.setName(extraInfo.getName());
407
        entity.setTypology(extraInfo.getTypology());
408
        entity.setProvenance(extraInfo.getProvenance());
409
        entity.setTrust(extraInfo.getTrust());
410
        entity.setValue(extraInfo.getValue());
411
        return entity;
412
    }
413

    
414
    public static OAIProvenance mapOAIProvenance(FieldTypeProtos.OAIProvenance oaiProvenance) {
415
        final OAIProvenance entity = new OAIProvenance();
416
        entity.setOriginDescription(mapOriginalDescription(oaiProvenance.getOriginDescription()));
417
        return entity;
418
    }
419

    
420
    public static OriginDescription mapOriginalDescription(FieldTypeProtos.OAIProvenance.OriginDescription originDescription) {
421
        final OriginDescription originDescriptionResult = new OriginDescription();
422
        originDescriptionResult.setHarvestDate(originDescription.getHarvestDate());
423
        originDescriptionResult.setAltered(originDescription.getAltered());
424
        originDescriptionResult.setBaseURL(originDescription.getBaseURL());
425
        originDescriptionResult.setIdentifier(originDescription.getIdentifier());
426
        originDescriptionResult.setDatestamp(originDescription.getDatestamp());
427
        originDescriptionResult.setMetadataNamespace(originDescription.getMetadataNamespace());
428
        return originDescriptionResult;
429
    }
430

    
431
    public static Field<String> mapStringField(FieldTypeProtos.StringField s) {
432
        final Field<String> stringField = new Field<>();
433
        stringField.setValue(s.getValue());
434
        stringField.setDataInfo(mapDataInfo(s.getDataInfo()));
435
        return stringField;
436
    }
437

    
438
    public static Field<Boolean> mapBoolField(FieldTypeProtos.BoolField b) {
439
        final Field<Boolean> booleanField = new Field<>();
440
        booleanField.setValue(b.getValue());
441
        booleanField.setDataInfo(mapDataInfo(b.getDataInfo()));
442
        return booleanField;
443
    }
444

    
445
    public static Field<Integer> mapIntField(FieldTypeProtos.IntField b) {
446
        final Field<Integer> entity = new Field<>();
447
        entity.setValue(b.getValue());
448
        entity.setDataInfo(mapDataInfo(b.getDataInfo()));
449
        return entity;
450
    }
451

    
452
    public static Journal mapJournal(FieldTypeProtos.Journal j) {
453
        final Journal journal = new Journal();
454
        journal.setConferencedate(j.getConferencedate());
455
        journal.setConferenceplace(j.getConferenceplace());
456
        journal.setEdition(j.getEdition());
457
        journal.setEp(j.getEp());
458
        journal.setIss(j.getIss());
459
        journal.setIssnLinking(j.getIssnLinking());
460
        journal.setIssnOnline(j.getIssnOnline());
461
        journal.setIssnPrinted(j.getIssnPrinted());
462
        journal.setName(j.getName());
463
        journal.setSp(j.getSp());
464
        journal.setVol(j.getVol());
465
        journal.setDataInfo(mapDataInfo(j.getDataInfo()));
466
        return journal;
467
    }
468

    
469
    public static Author mapAuthor(FieldTypeProtos.Author author) {
470
        final Author entity = new Author();
471
        entity.setFullname(author.getFullname());
472
        entity.setName(author.getName());
473
        entity.setSurname(author.getSurname());
474
        entity.setRank(author.getRank());
475
        entity.setPid(author.getPidList()
476
                .stream()
477
                .map(ProtoConverter::mapKV)
478
                .collect(Collectors.toList()));
479
        entity.setAffiliation(author.getAffiliationList()
480
                .stream()
481
                .map(ProtoConverter::mapStringField)
482
                .collect(Collectors.toList()));
483
        return entity;
484

    
485
    }
486

    
487
    public static GeoLocation mapGeolocation(ResultProtos.Result.GeoLocation geoLocation) {
488
        final GeoLocation entity = new GeoLocation();
489
        entity.setPoint(geoLocation.getPoint());
490
        entity.setBox(geoLocation.getBox());
491
        entity.setPlace(geoLocation.getPlace());
492
        return entity;
493
    }
494
}
(9-9/10)