Revision 58071
Added by Alessia Bardi about 4 years ago
| modules/dnet-mapreduce-jobs/trunk/src/test/java/eu/dnetlib/data/transform/XsltRowTransformerFactoryTest.java | ||
|---|---|---|
| 290 | 290 |
|
| 291 | 291 |
final List<Row> rows = Lists.newArrayList(); |
| 292 | 292 |
rows.addAll(asRows(loadFromTransformationProfile("oaf2hbase.xml"), load("odf.xml")));
|
| 293 |
// printAll(mapAll(buildTable(rows)));
|
|
| 293 |
printAll(mapAll(buildTable(rows))); |
|
| 294 | 294 |
} |
| 295 | 295 |
|
| 296 | 296 |
@Test |
| modules/dnet-mapreduce-jobs/trunk/src/test/resources/eu/dnetlib/data/transform/record.xml | ||
|---|---|---|
| 56 | 56 |
<oaf:collectedFrom id="opendoar____::296" name="Tesis Doctorals en Xarxa"/> |
| 57 | 57 |
<oaf:journal eissn="" ep="238" iss="February" issn="" sp="231" vol="1II">Constatin Brancusi University of Targu Jiu Annals - Economy Series</oaf:journal> |
| 58 | 58 |
<oaf:refereed>Peer-reviewed</oaf:refereed> |
| 59 |
<!-- <oaf:processingchargeamount currency="EUR">1721.47</oaf:processingchargeamount>--> |
|
| 59 | 60 |
</metadata> |
| 60 | 61 |
|
| 61 | 62 |
</record> |
| modules/dnet-mapreduce-jobs/trunk/src/test/resources/eu/dnetlib/data/transform/odf.xml | ||
|---|---|---|
| 26 | 26 |
<dc:creator>Corso, Mariano</dc:creator> |
| 27 | 27 |
<dc:title>(Re-)Designing the Business Model of a Digital Ecosystem: An Example in the Socio-Care Context</dc:title> |
| 28 | 28 |
<dc:date>2018</dc:date> |
| 29 |
<oaf:refereed>Peer-reviewed</oaf:refereed> |
|
| 29 | 30 |
</metadata> |
| 30 | 31 |
<provenance xmlns="http://www.openarchives.org/OAI/2.0/provenance" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/provenance http://www.openarchives.org/OAI/2.0/provenance.xsd"> |
| 31 | 32 |
<originDescription harvestDate="2019-04-11T14:51:27.828+02:00" altered="true"> |
| modules/dnet-mapreduce-jobs/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/dataexport/ProtoConverter.java | ||
|---|---|---|
| 83 | 83 |
i.setInstancetype(mapQualifier(ri.getInstancetype())); |
| 84 | 84 |
i.setLicense(mapStringField(ri.getLicense())); |
| 85 | 85 |
i.setUrl(ri.getUrlList()); |
| 86 |
//TODO: uncomment after model update |
|
| 87 |
//i.setRefereed(ri.getRefereed()); |
|
| 86 | 88 |
return i; |
| 87 | 89 |
} |
| 88 | 90 |
|
| ... | ... | |
| 376 | 378 |
.stream() |
| 377 | 379 |
.map(ProtoConverter::mapStringField) |
| 378 | 380 |
.collect(Collectors.toList())); |
| 379 |
entity.setRefereed(mapStringField(m.getRefereed())); |
|
| 380 | 381 |
entity.setContext(m.getContextList() |
| 381 | 382 |
.stream() |
| 382 | 383 |
.map(ProtoConverter::mapContext) |
| modules/dnet-mapreduce-jobs/trunk/src/main/java/eu/dnetlib/data/mapreduce/util/XmlRecordFactory.java | ||
|---|---|---|
| 67 | 67 |
protected Transformer transformer; |
| 68 | 68 |
|
| 69 | 69 |
protected static Predicate<String> instanceFilter = new Predicate<String>() {
|
| 70 |
final Set<String> instanceFieldFilter = Sets.newHashSet("instancetype", "hostedby", "license", "accessright", "collectedfrom", "dateofacceptance", "distributionlocation");
|
|
| 70 |
final Set<String> instanceFieldFilter = Sets.newHashSet("instancetype", "hostedby", "license", "accessright", "collectedfrom", "dateofacceptance", "distributionlocation", "refereed");
|
|
| 71 | 71 |
@Override |
| 72 | 72 |
public boolean apply(final String s) {
|
| 73 | 73 |
return instanceFieldFilter.contains(s); |
| modules/dnet-openaireplus-mapping-utils/trunk/src/main/java/eu/dnetlib/data/transform/xml/OafToHbaseXsltFunctions.java | ||
|---|---|---|
| 126 | 126 |
|
| 127 | 127 |
addStructuredProps(metadata, mDesc.findFieldByName("subject"), values.get("subject"), "keyword", "dnet:subject_classification_typologies");
|
| 128 | 128 |
addStructuredProps(metadata, mDesc.findFieldByName("title"), values.get("title"), "main title", "dnet:dataCite_title");
|
| 129 |
for (final String fieldname : Lists.newArrayList("description", "source", "contributor", "refereed")) {
|
|
| 129 |
for (final String fieldname : Lists.newArrayList("description", "source", "contributor")) {
|
|
| 130 | 130 |
if (values.get(fieldname) != null) {
|
| 131 | 131 |
for (final String s : values.get(fieldname).listValues()) {
|
| 132 | 132 |
addField(metadata, mDesc.findFieldByName(fieldname), s); |
| ... | ... | |
| 158 | 158 |
} |
| 159 | 159 |
} |
| 160 | 160 |
|
| 161 |
// #4468: processingchargeamount - implementatio to clarify: result vs instance. Single value vs repeatable |
|
| 162 |
// if (values.get("processingchargeamount") != null) {
|
|
| 163 |
// ElementList processingchargeamount = values.get("processingchargeamount");
|
|
| 164 |
// for(Element pc : processingchargeamount) {
|
|
| 165 |
// addField(metadata, mDesc.findFieldByName("processingchargeamount"), pc.getText());
|
|
| 166 |
// addField(metadata, mDesc.findFieldByName("processingchargecurrency"), pc.getAttributeValue("processingchargecurrency"));
|
|
| 167 |
// } |
|
| 168 |
// } |
|
| 169 | 161 |
return metadata; |
| 170 | 162 |
} |
| 171 | 163 |
|
| ... | ... | |
| 208 | 200 |
addField(instance, Instance.getDescriptor().findFieldByName("url"),
|
| 209 | 201 |
Lists.newArrayList(Iterables.filter(values.get("identifier").listValues(), urlFilter)));
|
| 210 | 202 |
} |
| 203 |
if (values.get("refereed") != null) {
|
|
| 204 |
addField(instance, Instance.getDescriptor().findFieldByName("refereed"), values.get("refereed").listValues());
|
|
| 205 |
} |
|
| 211 | 206 |
|
| 207 |
// #4468: processingchargeamount - implementatio to clarify: result vs instance. Single value vs repeatable |
|
| 208 |
// if (values.get("processingchargeamount") != null) {
|
|
| 209 |
// ElementList processingchargeamount = values.get("processingchargeamount");
|
|
| 210 |
// for(Element pc : processingchargeamount) {
|
|
| 211 |
// addField(instance, Instance.getDescriptor().findFieldByName("processingchargeamount"), pc.getText());
|
|
| 212 |
// addField(instance, Instance.getDescriptor().findFieldByName("processingchargecurrency"), pc.getAttributeValue("processingchargecurrency"));
|
|
| 213 |
// } |
|
| 214 |
// } |
|
| 215 |
|
|
| 212 | 216 |
result.addInstance(instance); |
| 213 | 217 |
|
| 214 | 218 |
final List<Element> extrefs = values.get("reference");
|
| modules/dnet-openaireplus-mapping-utils/trunk/src/main/java/eu/dnetlib/data/transform/xml/OdfToHbaseXsltFunctions.java | ||
|---|---|---|
| 352 | 352 |
|
| 353 | 353 |
addField(instance, Instance.getDescriptor().findFieldByName("collectedfrom"), collectedFroms);
|
| 354 | 354 |
addField(instance, Instance.getDescriptor().findFieldByName("dateofacceptance"), getFirstItem(dateaccepted));
|
| 355 |
if (values.get("refereed") != null) {
|
|
| 356 |
addField(instance, Instance.getDescriptor().findFieldByName("refereed"), values.get("refereed").listValues());
|
|
| 357 |
} |
|
| 355 | 358 |
|
| 356 | 359 |
result.addInstance(instance); |
| 357 | 360 |
|
| modules/dnet-openaireplus-mapping-utils/trunk/pom.xml | ||
|---|---|---|
| 3 | 3 |
<parent> |
| 4 | 4 |
<groupId>eu.dnetlib</groupId> |
| 5 | 5 |
<artifactId>dnet45-parent</artifactId> |
| 6 |
<version>1.0.0</version> |
|
| 6 |
<version>1.0.0-SNAPSHOT</version>
|
|
| 7 | 7 |
<relativePath /> |
| 8 | 8 |
</parent> |
| 9 | 9 |
<modelVersion>4.0.0</modelVersion> |
| ... | ... | |
| 86 | 86 |
<dependency> |
| 87 | 87 |
<groupId>eu.dnetlib</groupId> |
| 88 | 88 |
<artifactId>dnet-openaire-data-protos</artifactId> |
| 89 |
<version>[3.9.7]</version>
|
|
| 89 |
<version>[3.9.8-SNAPSHOT]</version>
|
|
| 90 | 90 |
</dependency> |
| 91 | 91 |
<dependency> |
| 92 | 92 |
<groupId>eu.dnetlib</groupId> |
| modules/dnet-openaire-data-protos/trunk/src/main/resources/eu/dnetlib/data/proto/Result.proto | ||
|---|---|---|
| 38 | 38 |
repeated StringField contributor = 30; |
| 39 | 39 |
optional Qualifier resourcetype = 19; |
| 40 | 40 |
repeated StringField coverage = 43; |
| 41 |
optional StringField refereed = 46; //peer-review status |
|
| 42 | 41 |
|
| 43 | 42 |
repeated Context context = 28; |
| 44 | 43 |
|
| ... | ... | |
| 65 | 64 |
repeated StringField contactgroup = 41; |
| 66 | 65 |
repeated StringField tool = 42; |
| 67 | 66 |
|
| 68 |
// ( article | book ) processing charges |
|
| 69 |
optional StringField processingchargeamount = 50; |
|
| 70 |
// currency - alphabetic code describe in ISO-4217 |
|
| 71 |
optional StringField processingchargecurrency = 51; |
|
| 72 |
|
|
| 73 | 67 |
} |
| 74 | 68 |
|
| 75 | 69 |
|
| ... | ... | |
| 94 | 88 |
|
| 95 | 89 |
optional KeyValue collectedfrom = 10; |
| 96 | 90 |
optional StringField dateofacceptance = 11; |
| 91 |
|
|
| 92 |
optional StringField refereed = 13; //peer-review status |
|
| 93 |
|
|
| 94 |
// ( article | book ) processing charges |
|
| 95 |
optional StringField processingchargeamount = 14; |
|
| 96 |
// currency - alphabetic code describe in ISO-4217 |
|
| 97 |
optional StringField processingchargecurrency = 15; |
|
| 97 | 98 |
} |
| 98 | 99 |
|
| 99 | 100 |
message ExternalReference {
|
Also available in: Unified diff
#4008: refereed field as property of the result instance and supported both from OAF and ODF.