Project

General

Profile

1
package eu.dnetlib.data.mapreduce.hbase.dataexport;
2

    
3
import com.googlecode.protobuf.format.JsonFormat;
4
import eu.dnetlib.data.proto.*;
5
import eu.dnetlib.dhp.schema.oaf.*;
6

    
7
import java.io.Serializable;
8
import java.util.stream.Collectors;
9

    
10
public class ProtoConverter implements Serializable {
11

    
12
    public static Oaf convert(OafProtos.Oaf oaf) {
13
        try {
14
            switch (oaf.getKind()) {
15
                case entity:
16
                    return convertEntity(oaf);
17
                case relation:
18
                    return convertRelation(oaf);
19
                default:
20
                    throw new IllegalArgumentException("invalid kind " + oaf.getKind());
21
            }
22
        } catch (Throwable e) {
23
            throw new RuntimeException("error on getting " + JsonFormat.printToString(oaf), e);
24
        }
25
    }
26

    
27
    private static Relation convertRelation(OafProtos.Oaf oaf) {
28
        final OafProtos.OafRel r = oaf.getRel();
29
        final Relation rel = new Relation();
30
        rel.setDataInfo(mapDataInfo(oaf.getDataInfo()));
31
        rel.setLastupdatetimestamp(oaf.getLastupdatetimestamp());
32
        rel.setSource(r.getSource());
33
        rel.setTarget(r.getTarget());
34
        rel.setRelType(r.getRelType().toString());
35
        rel.setSubRelType(r.getSubRelType().toString());
36
        rel.setRelClass(r.getRelClass());
37
        rel.setCollectedFrom(r.getCollectedfromCount() > 0 ?
38
                r.getCollectedfromList().stream()
39
                        .map(kv -> mapKV(kv))
40
                        .collect(Collectors.toList()) : null);
41
        return rel;
42
    }
43

    
44
    private static OafEntity convertEntity(OafProtos.Oaf oaf) {
45

    
46
        switch (oaf.getEntity().getType()) {
47
            case result:
48
                return convertResult(oaf);
49
            case project:
50
                return convertProject(oaf);
51
            case datasource:
52
                return convertDataSource(oaf);
53
            case organization:
54
                return convertOrganization(oaf);
55
            default:
56
                throw new RuntimeException("received unknown type");
57
        }
58
    }
59

    
60
    private static Organization convertOrganization(OafProtos.Oaf oaf) {
61
        final OrganizationProtos.Organization.Metadata m = oaf.getEntity().getOrganization().getMetadata();
62
        final Organization org = setOaf(new Organization(), oaf);
63
        setEntity(org, oaf);
64
        org.setLegalshortname(mapStringField(m.getLegalshortname()));
65
        org.setLegalname(mapStringField(m.getLegalname()));
66
        org.setAlternativeNames(m.getAlternativeNamesList().
67
                stream()
68
                .map(ProtoConverter::mapStringField)
69
                .collect(Collectors.toList()));
70
        org.setWebsiteurl(mapStringField(m.getWebsiteurl()));
71
        org.setLogourl(mapStringField(m.getLogourl()));
72
        org.setEclegalbody(mapStringField(m.getEclegalbody()));
73
        org.setEclegalperson(mapStringField(m.getEclegalperson()));
74
        org.setEcnonprofit(mapStringField(m.getEcnonprofit()));
75
        org.setEcresearchorganization(mapStringField(m.getEcresearchorganization()));
76
        org.setEchighereducation(mapStringField(m.getEchighereducation()));
77
        org.setEcinternationalorganizationeurinterests(mapStringField(m.getEcinternationalorganizationeurinterests()));
78
        org.setEcinternationalorganization(mapStringField(m.getEcinternationalorganization()));
79
        org.setEcenterprise(mapStringField(m.getEcenterprise()));
80
        org.setEcsmevalidated(mapStringField(m.getEcsmevalidated()));
81
        org.setEcnutscode(mapStringField(m.getEcnutscode()));
82
        org.setCountry(mapQualifier(m.getCountry()));
83

    
84
        return org;
85
    }
86

    
87
    private static Datasource convertDataSource(OafProtos.Oaf oaf) {
88
        final DatasourceProtos.Datasource.Metadata m = oaf.getEntity().getDatasource().getMetadata();
89
        final Datasource datasource = setOaf(new Datasource(), oaf);
90
        setEntity(datasource, oaf);
91
        datasource.setAccessinfopackage(m.getAccessinfopackageList()
92
                .stream()
93
                .map(ProtoConverter::mapStringField)
94
                .collect(Collectors.toList()));
95
        datasource.setCertificates(mapStringField(m.getCertificates()));
96
        datasource.setCitationguidelineurl(mapStringField(m.getCitationguidelineurl()));
97
        datasource.setContactemail(mapStringField(m.getContactemail()));
98
        datasource.setDatabaseaccessrestriction(mapStringField(m.getDatabaseaccessrestriction()));
99
        datasource.setDatabaseaccesstype(mapStringField(m.getDatabaseaccesstype()));
100
        datasource.setDataprovider(mapBoolField(m.getDataprovider()));
101
        datasource.setDatasourcetype(mapQualifier(m.getDatasourcetype()));
102
        datasource.setDatauploadrestriction(mapStringField(m.getDatauploadrestriction()));
103
        datasource.setCitationguidelineurl(mapStringField(m.getCitationguidelineurl()));
104
        datasource.setDatauploadtype(mapStringField(m.getDatauploadtype()));
105
        datasource.setDateofvalidation(mapStringField(m.getDateofvalidation()));
106
        datasource.setDescription(mapStringField(m.getDescription()));
107
        datasource.setEnglishname(mapStringField(m.getEnglishname()));
108
        datasource.setLatitude(mapStringField(m.getLatitude()));
109
        datasource.setLongitude(mapStringField(m.getLongitude()));
110
        datasource.setLogourl(mapStringField(m.getLogourl()));
111
        datasource.setMissionstatementurl(mapStringField(m.getMissionstatementurl()));
112
        datasource.setNamespaceprefix(mapStringField(m.getNamespaceprefix()));
113
        datasource.setOdcontenttypes(m.getOdcontenttypesList()
114
                .stream()
115
                .map(ProtoConverter::mapStringField)
116
                .collect(Collectors.toList()));
117
        datasource.setOdlanguages(m.getOdlanguagesList()
118
                .stream()
119
                .map(ProtoConverter::mapStringField)
120
                .collect(Collectors.toList()));
121
        datasource.setOdnumberofitems(mapStringField(m.getOdnumberofitems()));
122
        datasource.setOdnumberofitemsdate(mapStringField(m.getOdnumberofitemsdate()));
123
        datasource.setOdpolicies(mapStringField(m.getOdpolicies()));
124
        datasource.setOfficialname(mapStringField(m.getOfficialname()));
125
        datasource.setOpenairecompatibility(mapQualifier(m.getOpenairecompatibility()));
126
        datasource.setPidsystems(mapStringField(m.getPidsystems()));
127
        datasource.setPolicies(m.getPoliciesList()
128
                .stream()
129
                .map(ProtoConverter::mapKV)
130
                .collect(Collectors.toList()));
131
        datasource.setQualitymanagementkind(mapStringField(m.getQualitymanagementkind()));
132
        datasource.setReleaseenddate(mapStringField(m.getReleaseenddate()));
133
        datasource.setServiceprovider(mapBoolField(m.getServiceprovider()));
134
        datasource.setReleasestartdate(mapStringField(m.getReleasestartdate()));
135
        datasource.setSubjects(m.getSubjectsList()
136
                .stream()
137
                .map(ProtoConverter::mapStructuredProperty)
138
                .collect(Collectors.toList()));
139
        datasource.setVersioning(mapBoolField(m.getVersioning()));
140
        datasource.setWebsiteurl(mapStringField(m.getWebsiteurl()));
141
        datasource.setJournal(mapJournal(m.getJournal()));
142

    
143

    
144
        return datasource;
145
    }
146

    
147
    private static Project convertProject(OafProtos.Oaf oaf) {
148
        final ProjectProtos.Project.Metadata m = oaf.getEntity().getProject().getMetadata();
149
        final Project project = setOaf(new Project(), oaf);
150
        setEntity(project, oaf);
151
        project.setAcronym(mapStringField(m.getAcronym()));
152
        project.setCallidentifier(mapStringField(m.getCallidentifier()));
153
        project.setCode(mapStringField(m.getCode()));
154
        project.setContactemail(mapStringField(m.getContactemail()));
155
        project.setContactfax(mapStringField(m.getContactfax()));
156
        project.setContactfullname(mapStringField(m.getContactfullname()));
157
        project.setContactphone(mapStringField(m.getContactphone()));
158
        project.setContracttype(mapQualifier(m.getContracttype()));
159
        project.setCurrency(mapStringField(m.getCurrency()));
160
        project.setDuration(mapStringField(m.getDuration()));
161
        project.setEcarticle29_3(mapStringField(m.getEcarticle293()));
162
        project.setEcsc39(mapStringField(m.getEcsc39()));
163
        project.setOamandatepublications(mapStringField(m.getOamandatepublications()));
164
        project.setStartdate(mapStringField(m.getStartdate()));
165
        project.setEnddate(mapStringField(m.getEnddate()));
166
        project.setFundedamount(m.getFundedamount());
167
        project.setTotalcost(m.getTotalcost());
168
        project.setKeywords(mapStringField(m.getKeywords()));
169
        project.setSubjects(m.getSubjectsList().stream()
170
                .map(sp -> mapStructuredProperty(sp))
171
                .collect(Collectors.toList()));
172
        project.setTitle(mapStringField(m.getTitle()));
173
        project.setWebsiteurl(mapStringField(m.getWebsiteurl()));
174
        project.setFundingtree(m.getFundingtreeList().stream()
175
                .map(f -> mapStringField(f))
176
                .collect(Collectors.toList()));
177
        project.setJsonextrainfo(mapStringField(m.getJsonextrainfo()));
178
        project.setSummary(mapStringField(m.getSummary()));
179
        project.setOptional1(mapStringField(m.getOptional1()));
180
        project.setOptional2(mapStringField(m.getOptional2()));
181
        return project;
182
    }
183

    
184
    private static Result convertResult(OafProtos.Oaf oaf) {
185
        switch (oaf.getEntity().getResult().getMetadata().getResulttype().getClassid()) {
186

    
187
            case "dataset":
188
                return createDataset(oaf);
189
            case "publication":
190
                return createPublication(oaf);
191
            case "software":
192
                return createSoftware(oaf);
193
            case "other":
194
                return createORP(oaf);
195
            default:
196
                throw new RuntimeException("received unknown type: " + oaf.getEntity().getResult().getMetadata().getResulttype().getClassid());
197
        }
198
    }
199

    
200
    private static Software createSoftware(OafProtos.Oaf oaf) {
201
        ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata();
202
        Software software = setOaf(new Software(), oaf);
203
        setEntity(software, oaf);
204
        setResult(software, oaf);
205

    
206
        software.setDocumentationUrl(m.getDocumentationUrlList()
207
                .stream()
208
                .map(ProtoConverter::mapStringField)
209
                .collect(Collectors.toList()));
210
        software.setLicense(m.getLicenseList()
211
                .stream()
212
                .map(ProtoConverter::mapStructuredProperty)
213
                .collect(Collectors.toList()));
214
        software.setCodeRepositoryUrl(mapStringField(m.getCodeRepositoryUrl()));
215
        software.setProgrammingLanguage(mapQualifier(m.getProgrammingLanguage()));
216
        return software;
217
    }
218

    
219
    private static OtherResearchProduct createORP(OafProtos.Oaf oaf) {
220
        ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata();
221
        OtherResearchProduct otherResearchProducts = setOaf(new OtherResearchProduct(), oaf);
222
        setEntity(otherResearchProducts, oaf);
223
        setResult(otherResearchProducts, oaf);
224
        otherResearchProducts.setContactperson(m.getContactpersonList()
225
                .stream()
226
                .map(ProtoConverter::mapStringField)
227
                .collect(Collectors.toList()));
228
        otherResearchProducts.setContactgroup(m.getContactgroupList()
229
                .stream()
230
                .map(ProtoConverter::mapStringField)
231
                .collect(Collectors.toList()));
232
        otherResearchProducts.setTool(m.getToolList()
233
                .stream()
234
                .map(ProtoConverter::mapStringField)
235
                .collect(Collectors.toList()));
236

    
237
        return otherResearchProducts;
238
    }
239

    
240
    private static Publication createPublication(OafProtos.Oaf oaf) {
241

    
242
        ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata();
243
        Publication publication = setOaf(new Publication(), oaf);
244
        setEntity(publication, oaf);
245
        setResult(publication, oaf);
246
        publication.setJournal(mapJournal(m.getJournal()));
247
        return publication;
248
    }
249

    
250
    private static Dataset createDataset(OafProtos.Oaf oaf) {
251

    
252
        ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata();
253
        Dataset dataset = setOaf(new Dataset(), oaf);
254
        setEntity(dataset, oaf);
255
        setResult(dataset, oaf);
256
        dataset.setStoragedate(mapStringField(m.getStoragedate()));
257
        dataset.setDevice(mapStringField(m.getDevice()));
258
        dataset.setSize(mapStringField(m.getSize()));
259
        dataset.setVersion(mapStringField(m.getVersion()));
260
        dataset.setLastmetadataupdate(mapStringField(m.getLastmetadataupdate()));
261
        dataset.setMetadataversionnumber(mapStringField(m.getMetadataversionnumber()));
262
        dataset.setGeolocation(m.getGeolocationList()
263
                .stream()
264
                .map(ProtoConverter::mapGeolocation)
265
                .collect(Collectors.toList()));
266
        return dataset;
267

    
268
    }
269

    
270
    public static <T extends Oaf> T setOaf(T oaf, OafProtos.Oaf o) {
271
        oaf.setDataInfo(mapDataInfo(o.getDataInfo()));
272
        oaf.setLastupdatetimestamp(o.getLastupdatetimestamp());
273
        return oaf;
274
    }
275

    
276
    public static <T extends OafEntity> T setEntity(T entity, OafProtos.Oaf oaf) {
277
        //setting Entity fields
278
        final OafProtos.OafEntity e = oaf.getEntity();
279
        entity.setId(e.getId());
280
        entity.setOriginalId(e.getOriginalIdList());
281
        entity.setCollectedfrom(e.getCollectedfromList()
282
                .stream()
283
                .map(ProtoConverter::mapKV)
284
                .collect(Collectors.toList()));
285
        entity.setPid(e.getPidList().stream()
286
                .map(ProtoConverter::mapStructuredProperty)
287
                .collect(Collectors.toList()));
288
        entity.setDateofcollection(entity.getDateofcollection());
289
        entity.setDateoftransformation(entity.getDateoftransformation());
290
        entity.setExtraInfo(e.getExtraInfoList()
291
                .stream()
292
                .map(ProtoConverter::mapExtraInfo)
293
                .collect(Collectors.toList()));
294
        return entity;
295
    }
296

    
297
    public static <T extends Result> T setResult(T entity, OafProtos.Oaf oaf) {
298
        //setting Entity fields
299
        final ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata();
300
        entity.setAuthor(m.getAuthorList()
301
                .stream()
302
                .map(ProtoConverter::mapAuthor)
303
                .collect(Collectors.toList()));
304
        entity.setResulttype(mapQualifier(m.getResulttype()));
305
        entity.setLanguage(mapQualifier(m.getLanguage()));
306
        entity.setCountry(m.getCountryList()
307
                .stream()
308
                .map(ProtoConverter::mapQualifier)
309
                .collect(Collectors.toList()));
310
        entity.setSubject(m.getSubjectList()
311
                .stream()
312
                .map(ProtoConverter::mapStructuredProperty)
313
                .collect(Collectors.toList()));
314
        entity.setTitle(m.getTitleList()
315
                .stream()
316
                .map(ProtoConverter::mapStructuredProperty)
317
                .collect(Collectors.toList()));
318
        entity.setRelevantdate(m.getRelevantdateList()
319
                .stream()
320
                .map(ProtoConverter::mapStructuredProperty)
321
                .collect(Collectors.toList()));
322
        entity.setDescription(m.getDescriptionList()
323
                .stream()
324
                .map(ProtoConverter::mapStringField)
325
                .collect(Collectors.toList()));
326
        entity.setDateofacceptance(mapStringField(m.getDateofacceptance()));
327
        entity.setPublisher(mapStringField(m.getPublisher()));
328
        entity.setEmbargoenddate(mapStringField(m.getEmbargoenddate()));
329
        entity.setSource(m.getSourceList()
330
                .stream()
331
                .map(ProtoConverter::mapStringField)
332
                .collect(Collectors.toList()));
333
        entity.setFulltext(m.getFulltextList()
334
                .stream()
335
                .map(ProtoConverter::mapStringField)
336
                .collect(Collectors.toList()));
337
        entity.setFormat(m.getFormatList()
338
                .stream()
339
                .map(ProtoConverter::mapStringField)
340
                .collect(Collectors.toList()));
341
        entity.setContributor(m.getContributorList()
342
                .stream()
343
                .map(ProtoConverter::mapStringField)
344
                .collect(Collectors.toList()));
345
        entity.setResourcetype(mapQualifier(m.getResourcetype()));
346
        entity.setCoverage(m.getCoverageList()
347
                .stream()
348
                .map(ProtoConverter::mapStringField)
349
                .collect(Collectors.toList()));
350
        entity.setRefereed(mapStringField(m.getRefereed()));
351
        entity.setContext(m.getContextList()
352
                .stream()
353
                .map(ProtoConverter::mapContext)
354
                .collect(Collectors.toList()));
355

    
356
        return entity;
357
    }
358

    
359
    private static Context mapContext(ResultProtos.Result.Context context) {
360

    
361
        final Context entity = new Context();
362
        entity.setId(context.getId());
363
        entity.setDataInfo(context.getDataInfoList()
364
                .stream()
365
                .map(ProtoConverter::mapDataInfo)
366
                .collect(Collectors.toList()));
367
        return entity;
368
    }
369

    
370

    
371
    public static KeyValue mapKV(FieldTypeProtos.KeyValue kv) {
372
        final KeyValue keyValue = new KeyValue();
373
        keyValue.setKey(kv.getKey());
374
        keyValue.setValue(kv.getValue());
375
        keyValue.setDataInfo(mapDataInfo(kv.getDataInfo()));
376
        return keyValue;
377
    }
378

    
379
    public static DataInfo mapDataInfo(FieldTypeProtos.DataInfo d) {
380
        final DataInfo dataInfo = new DataInfo();
381
        dataInfo.setDeletedbyinference(d.getDeletedbyinference());
382
        dataInfo.setInferenceprovenance(d.getInferenceprovenance());
383
        dataInfo.setInferred(d.getInferred());
384
        dataInfo.setInvisible(d.getInvisible());
385
        dataInfo.setProvenanceaction(mapQualifier(d.getProvenanceaction()));
386
        return dataInfo;
387
    }
388

    
389
    public static Qualifier mapQualifier(FieldTypeProtos.Qualifier q) {
390
        final Qualifier qualifier = new Qualifier();
391
        qualifier.setClassid(q.getClassid());
392
        qualifier.setClassname(q.getClassname());
393
        qualifier.setSchemeid(q.getSchemeid());
394
        qualifier.setSchemename(q.getSchemename());
395
        return qualifier;
396
    }
397

    
398
    public static StructuredProperty mapStructuredProperty(FieldTypeProtos.StructuredProperty sp) {
399
        final StructuredProperty structuredProperty = new StructuredProperty();
400
        structuredProperty.setValue(sp.getValue());
401
        structuredProperty.setQualifier(mapQualifier(sp.getQualifier()));
402
        structuredProperty.setDataInfo(mapDataInfo(sp.getDataInfo()));
403
        return structuredProperty;
404
    }
405

    
406
    public static ExtraInfo mapExtraInfo(FieldTypeProtos.ExtraInfo extraInfo) {
407
        final ExtraInfo entity = new ExtraInfo();
408
        entity.setName(extraInfo.getName());
409
        entity.setTypology(extraInfo.getTypology());
410
        entity.setProvenance(extraInfo.getProvenance());
411
        entity.setTrust(extraInfo.getTrust());
412
        entity.setValue(extraInfo.getValue());
413
        return entity;
414
    }
415

    
416
    public static OAIProvenance mapOAIProvenance(FieldTypeProtos.OAIProvenance oaiProvenance) {
417
        final OAIProvenance entity = new OAIProvenance();
418
        entity.setOriginDescription(mapOriginalDescription(oaiProvenance.getOriginDescription()));
419
        return entity;
420
    }
421

    
422
    public static OriginDescription mapOriginalDescription(FieldTypeProtos.OAIProvenance.OriginDescription originDescription) {
423
        final OriginDescription originDescriptionResult = new OriginDescription();
424
        originDescriptionResult.setHarvestDate(originDescription.getHarvestDate());
425
        originDescriptionResult.setAltered(originDescription.getAltered());
426
        originDescriptionResult.setBaseURL(originDescription.getBaseURL());
427
        originDescriptionResult.setIdentifier(originDescription.getIdentifier());
428
        originDescriptionResult.setDatestamp(originDescription.getDatestamp());
429
        originDescriptionResult.setMetadataNamespace(originDescription.getMetadataNamespace());
430
        return originDescriptionResult;
431
    }
432

    
433
    public static Field<String> mapStringField(FieldTypeProtos.StringField s) {
434
        final Field<String> stringField = new Field<>();
435
        stringField.setValue(s.getValue());
436
        stringField.setDataInfo(mapDataInfo(s.getDataInfo()));
437
        return stringField;
438
    }
439

    
440
    public static Field<Boolean> mapBoolField(FieldTypeProtos.BoolField b) {
441
        final Field<Boolean> booleanField = new Field<>();
442
        booleanField.setValue(b.getValue());
443
        booleanField.setDataInfo(mapDataInfo(b.getDataInfo()));
444
        return booleanField;
445
    }
446

    
447
    public static Field<Integer> mapIntField(FieldTypeProtos.IntField b) {
448
        final Field<Integer> entity = new Field<>();
449
        entity.setValue(b.getValue());
450
        entity.setDataInfo(mapDataInfo(b.getDataInfo()));
451
        return entity;
452
    }
453

    
454
    public static Journal mapJournal(FieldTypeProtos.Journal j) {
455
        final Journal journal = new Journal();
456
        journal.setConferencedate(j.getConferencedate());
457
        journal.setConferenceplace(j.getConferenceplace());
458
        journal.setEdition(j.getEdition());
459
        journal.setEp(j.getEp());
460
        journal.setIss(j.getIss());
461
        journal.setIssnLinking(j.getIssnLinking());
462
        journal.setIssnOnline(j.getIssnOnline());
463
        journal.setIssnPrinted(j.getIssnPrinted());
464
        journal.setName(j.getName());
465
        journal.setSp(j.getSp());
466
        journal.setVol(j.getVol());
467
        journal.setDataInfo(mapDataInfo(j.getDataInfo()));
468
        return journal;
469
    }
470

    
471
    public static Author mapAuthor(FieldTypeProtos.Author author) {
472
        final Author entity = new Author();
473
        entity.setFullname(author.getFullname());
474
        entity.setName(author.getName());
475
        entity.setSurname(author.getSurname());
476
        entity.setRank(author.getRank());
477
        entity.setPid(author.getPidList()
478
                .stream()
479
                .map(ProtoConverter::mapKV)
480
                .collect(Collectors.toList()));
481
        entity.setAffiliation(author.getAffiliationList()
482
                .stream()
483
                .map(ProtoConverter::mapStringField)
484
                .collect(Collectors.toList()));
485
        return entity;
486

    
487
    }
488

    
489
    public static GeoLocation mapGeolocation(ResultProtos.Result.GeoLocation geoLocation) {
490
        final GeoLocation entity = new GeoLocation();
491
        entity.setPoint(geoLocation.getPoint());
492
        entity.setBox(geoLocation.getBox());
493
        entity.setPlace(geoLocation.getPlace());
494
        return entity;
495
    }
496
}
(9-9/10)