Revision 56505
Added by Antonis Lempesis almost 5 years ago
| modules/dnet-openaire-stats-export-wf/trunk/dnet-openaire-stats/src/main/java/eu/dnetlib/data/mapreduce/hbase/statsExport/utils/Serializer.java | ||
|---|---|---|
| 68 | 68 |
case result: |
| 69 | 69 |
|
| 70 | 70 |
return buildResult(oaf); |
| 71 |
/* |
|
| 72 |
case person: |
|
| 73 |
return buildPerson(oaf, DELIM, ENCLOSING); |
|
| 74 |
*/ |
|
| 75 | 71 |
default: |
| 76 | 72 |
break; |
| 77 | 73 |
} |
| 78 | 74 |
break; |
| 79 | 75 |
case relation: |
| 80 |
|
|
| 81 | 76 |
return buildRel(oaf.getRel()); |
| 77 |
} |
|
| 82 | 78 |
|
| 83 |
} |
|
| 84 | 79 |
return null; |
| 85 | 80 |
} |
| 86 | 81 |
|
| ... | ... | |
| 95 | 90 |
} |
| 96 | 91 |
|
| 97 | 92 |
private String buildRel(OafRel Rel) {
|
| 98 |
|
|
| 99 | 93 |
return cleanId(Rel.getTarget()) + DELIM; |
| 100 |
|
|
| 101 | 94 |
} |
| 102 | 95 |
|
| 103 | 96 |
public void extractRelations(Oaf oaf, Multimap<String, String> relations) {
|
| ... | ... | |
| 107 | 100 |
switch (valueEntity.getType()) {
|
| 108 | 101 |
case datasource: |
| 109 | 102 |
getDatasourceLanguages(valueEntity, relations); |
| 110 |
// getDatasourceWebsite(valueEntity, relations); |
|
| 111 | 103 |
case result: |
| 112 | 104 |
getResultTopics(valueEntity, relations); |
| 113 | 105 |
getResultLanguages(valueEntity, relations); |
| ... | ... | |
| 116 | 108 |
getResultConcepts(valueEntity, relations); |
| 117 | 109 |
getResultDois(valueEntity, relations); |
| 118 | 110 |
getResultCitations(valueEntity, relations); |
| 119 |
// getResultDescriptions(valueEntity, relations, DELIM, ENCLOSING); |
|
| 120 |
// getResultExtra(valueEntity, relations, DELIM, ENCLOSING); |
|
| 121 | 111 |
|
| 122 | 112 |
case project: |
| 123 | 113 |
getProjectKeywords(valueEntity, relations); |
| ... | ... | |
| 128 | 118 |
|
| 129 | 119 |
} |
| 130 | 120 |
|
| 131 |
private void getDatasourceWebsite(OafEntity valueEntity, Multimap<String, String> relations) {
|
|
| 132 |
Datasource d = valueEntity.getDatasource(); |
|
| 133 |
Metadata metadata = d.getMetadata(); |
|
| 134 |
|
|
| 135 |
relations.put("datasourceWebsite", getStringField(metadata.getWebsiteurl().getValue()));
|
|
| 136 |
} |
|
| 137 |
|
|
| 138 |
private void getResultExtra(OafEntity valueEntity, Multimap<String, String> relations) {
|
|
| 139 |
Result result = valueEntity.getResult(); |
|
| 140 |
Result.Metadata metadata = result.getMetadata(); |
|
| 141 |
|
|
| 142 |
StringBuilder buff = new StringBuilder(); |
|
| 143 |
String titleString = ""; |
|
| 144 |
|
|
| 145 |
for (int i = 0; i < metadata.getTitleList().size(); i++) {
|
|
| 146 |
StructuredProperty title = metadata.getTitleList().get(i); |
|
| 147 |
|
|
| 148 |
titleString = title.getValue().replaceAll("\\s+", " ");
|
|
| 149 |
titleString = titleString.replaceAll("\n", " ");
|
|
| 150 |
break; |
|
| 151 |
} |
|
| 152 |
|
|
| 153 |
// pubtitle |
|
| 154 |
buff.append(getStringField(titleString)); |
|
| 155 |
|
|
| 156 |
String sources = ""; |
|
| 157 |
for (Instance instance : (result.getInstanceList())) {
|
|
| 158 |
List<String> urls = instance.getUrlList(); |
|
| 159 |
for (String url : urls) {
|
|
| 160 |
sources += cleanUrl(url) + " ;"; |
|
| 161 |
} |
|
| 162 |
} |
|
| 163 |
|
|
| 164 |
//sources |
|
| 165 |
sources = ENCLOSING + sources + ENCLOSING + DELIM; |
|
| 166 |
buff.append(sources); |
|
| 167 |
|
|
| 168 |
relations.put("resultExtra", buff.toString());
|
|
| 169 |
} |
|
| 170 |
|
|
| 171 | 121 |
private void getOriginalId(OafEntity oafEntity, Multimap<String, String> relations) {
|
| 172 | 122 |
|
| 173 | 123 |
String relName = oafEntity.getType().toString().toLowerCase() + "Oid"; |
| ... | ... | |
| 260 | 210 |
private void getResultDatasources(OafEntity valueEntity, Multimap<String, String> rels) {
|
| 261 | 211 |
Result result = valueEntity.getResult(); |
| 262 | 212 |
|
| 263 |
//TODO hosted by
|
|
| 213 |
// hosted by
|
|
| 264 | 214 |
for (Instance instance : (result.getInstanceList())) {
|
| 265 | 215 |
String hostedBy = instance.getHostedby().getKey(); |
| 266 | 216 |
|
| ... | ... | |
| 269 | 219 |
} |
| 270 | 220 |
} |
| 271 | 221 |
|
| 272 |
//TODO collected froms
|
|
| 222 |
// collected from
|
|
| 273 | 223 |
for (FieldTypeProtos.KeyValue collectedFromValue : (valueEntity.getCollectedfromList())) {
|
| 274 | 224 |
String collectedFrom = collectedFromValue.getKey(); |
| 275 | 225 |
|
| ... | ... | |
| 314 | 264 |
} catch (Exception e) {
|
| 315 | 265 |
|
| 316 | 266 |
} |
| 317 |
|
|
| 318 |
/* |
|
| 319 |
rels.put("resultCitation", getStringField(extraInfo.getTrust(), DELIM, ENCLOSING) +
|
|
| 320 |
getStringField(extraInfo.getProvenance(), DELIM, ENCLOSING) + getStringField(extraInfo.getValue(), DELIM, ENCLOSING)); |
|
| 321 |
*/ |
|
| 322 | 267 |
} |
| 323 |
|
|
| 324 | 268 |
} |
| 325 | 269 |
} |
| 326 | 270 |
|
| ... | ... | |
| 430 | 374 |
|
| 431 | 375 |
// `authors`, |
| 432 | 376 |
int authors = metadata.getAuthorCount(); |
| 433 |
|
|
| 434 |
|
|
| 435 | 377 |
String delayed = "no"; |
| 436 | 378 |
|
| 437 | 379 |
for (OafRel rel : oaf.getEntity().getCachedRelList()) {
|
| 438 |
|
|
| 439 |
/* |
|
| 440 |
if (rel.getRelType().equals(RelType.personResult)) {
|
|
| 441 |
|
|
| 442 |
authors++; |
|
| 443 |
} else |
|
| 444 |
*/ |
|
| 445 | 380 |
if (rel.getRelType().equals(RelType.resultProject)) |
| 446 |
// TODO remember : in result Project, first id is project, second is |
|
| 447 |
// result. |
|
| 381 |
// remember : in result Project, first id is project, second is result. |
|
| 448 | 382 |
{
|
| 449 | 383 |
String daysfromend = getYearDifferenceInteger(rel.getResultProject().getOutcome().getRelMetadata().getEnddate(), |
| 450 | 384 |
rel.getResultProject().getOutcome().getRelMetadata().getStartdate()); |
| ... | ... | |
| 478 | 412 |
|
| 479 | 413 |
} |
| 480 | 414 |
|
| 481 |
|
|
| 482 |
/* |
|
| 483 |
private static String getBestLicense(Result result) {
|
|
| 484 |
Qualifier bestLicense = null; |
|
| 485 |
LicenseComparator lc = new LicenseComparator(); |
|
| 486 |
for (Instance instance : (result.getInstanceList())) {
|
|
| 487 |
if (lc.compare(bestLicense, instance.getLicence()) > 0) {
|
|
| 488 |
bestLicense = instance.getLicence(); |
|
| 489 |
} |
|
| 490 |
} |
|
| 491 |
if (bestLicense != null) {
|
|
| 492 |
return bestLicense.getClassname(); |
|
| 493 |
} else {
|
|
| 494 |
return null; |
|
| 495 |
} |
|
| 496 |
} |
|
| 497 |
|
|
| 498 |
// TODO here iterate over all values |
|
| 499 |
private static String getAccessMode(Result result) {
|
|
| 500 |
String accessMode = " "; |
|
| 501 |
for (Instance instance : (result.getInstanceList())) {
|
|
| 502 |
if (instance.getLicence().getClassname() != null && !instance.getLicence().getClassname().isEmpty()) {
|
|
| 503 |
accessMode = instance.getLicence().getClassname(); |
|
| 504 |
break; |
|
| 505 |
} |
|
| 506 |
|
|
| 507 |
} |
|
| 508 |
|
|
| 509 |
return accessMode; |
|
| 510 |
} |
|
| 511 |
*/ |
|
| 512 |
|
|
| 513 | 415 |
private String getBestLicense(Result result) {
|
| 514 | 416 |
Qualifier bestLicense = null; |
| 515 | 417 |
LicenseComparator lc = new LicenseComparator(); |
| ... | ... | |
| 525 | 427 |
} |
| 526 | 428 |
} |
| 527 | 429 |
|
| 528 |
// TODO here iterate over all values |
|
| 529 |
private String getAccessMode(Result result) {
|
|
| 530 |
String accessMode = " "; |
|
| 531 |
for (Instance instance : (result.getInstanceList())) {
|
|
| 532 |
if (instance.getAccessright().getClassname() != null && !instance.getAccessright().getClassname().isEmpty()) {
|
|
| 533 |
accessMode = instance.getAccessright().getClassname(); |
|
| 534 |
break; |
|
| 535 |
} |
|
| 536 |
|
|
| 537 |
} |
|
| 538 |
|
|
| 539 |
return accessMode; |
|
| 540 |
} |
|
| 541 |
|
|
| 542 | 430 |
private String buildProject(Oaf oaf) {
|
| 543 | 431 |
|
| 544 | 432 |
FundingParser fundingParser = new FundingParser(DELIM, ENCLOSING); |
| ... | ... | |
| 559 | 447 |
List<StringField> fundList = metadata.getFundingtreeList(); |
| 560 | 448 |
if (!fundList.isEmpty()) // `funding_lvl0`, |
| 561 | 449 |
{
|
| 562 |
//TODO funder + 3 funding levels |
|
| 563 |
/* funder text, |
|
| 564 |
funding_lvl0 text, |
|
| 565 |
funding_lvl1 text, |
|
| 566 |
funding_lvl2 text, |
|
| 567 |
funding_lvl3 text,*/ |
|
| 450 |
//funder + 3 funding levels |
|
| 568 | 451 |
buff.append(fundingParser.getFundingInfo(fundList.get(0).getValue())); |
| 569 | 452 |
} else {
|
| 570 | 453 |
buff.append(fundingParser.getFundingInfo(""));
|
| ... | ... | |
| 618 | 501 |
return buff.toString(); |
| 619 | 502 |
} |
| 620 | 503 |
|
| 621 |
/* |
|
| 622 |
private static String buildPerson(Oaf oaf, String DELIM, String ENCLOSING) {
|
|
| 623 |
StringBuilder buff = new StringBuilder(); |
|
| 624 | 504 |
|
| 625 |
PersonProtos.Person.Metadata metadata = oaf.getEntity().getPerson().getMetadata(); |
|
| 626 |
|
|
| 627 |
// `person_id`, |
|
| 628 |
//buff.append(cleanId(oaf.getEntity().getId(), DELIM, ENCLOSING) + DELIM); |
|
| 629 |
|
|
| 630 |
|
|
| 631 |
// person_result |
|
| 632 |
buff.append(cleanId(oaf.getEntity().getId(), DELIM, ENCLOSING) + DELIM); |
|
| 633 |
|
|
| 634 |
|
|
| 635 |
//firstName |
|
| 636 |
buff.append(getStringField(metadata.getFirstname().getValue(), DELIM, ENCLOSING)); |
|
| 637 |
|
|
| 638 |
//LastName |
|
| 639 |
|
|
| 640 |
String secondName = new String(); |
|
| 641 |
for (StringField s : metadata.getSecondnamesList()) {
|
|
| 642 |
secondName += s.getValue().replace("\n", " ").replace(",", " ") + " ";
|
|
| 643 |
} |
|
| 644 |
|
|
| 645 |
buff.append(getStringField(secondName, DELIM, ENCLOSING)); |
|
| 646 |
|
|
| 647 |
// `fullname`, |
|
| 648 |
buff.append(getStringField(metadata.getFullname().getValue(), DELIM, ENCLOSING)); |
|
| 649 |
|
|
| 650 |
// `Nationality`, |
|
| 651 |
buff.append(getStringField(metadata.getNationality().getClassid(), DELIM, ENCLOSING)); |
|
| 652 |
|
|
| 653 |
// `Email`, |
|
| 654 |
buff.append(getStringField(metadata.getEmail().getValue(), DELIM, ENCLOSING)); |
|
| 655 |
|
|
| 656 |
// `Phone`, |
|
| 657 |
buff.append(getStringField(metadata.getPhone().getValue(), DELIM, ENCLOSING)); |
|
| 658 |
|
|
| 659 |
//deletedByInference |
|
| 660 |
|
|
| 661 |
buff.append(getStringField(String.valueOf(oaf.getDataInfo().getDeletedbyinference()), DELIM, ENCLOSING)); |
|
| 662 |
|
|
| 663 |
// `number`, |
|
| 664 |
buff.append(getStringField("1", DELIM, ENCLOSING));
|
|
| 665 |
|
|
| 666 |
return buff.toString(); |
|
| 667 |
|
|
| 668 |
} |
|
| 669 |
*/ |
|
| 670 |
|
|
| 671 |
|
|
| 672 | 505 |
private String getYearDifferenceInteger(String enddate, String startdate) {
|
| 673 | 506 |
|
| 674 | 507 |
if (enddate != null && !enddate.isEmpty() && startdate != null && !startdate.isEmpty()) {
|
| ... | ... | |
| 790 | 623 |
value = value.replace(DELIM, " "); |
| 791 | 624 |
value = value.replace(ENCLOSING, " "); |
| 792 | 625 |
value = value.replaceAll("\\r\\n|\\r|\\n", " ");
|
| 793 |
//value = value.replace("\"", "");
|
|
| 794 |
//value = value.replace("'", "");
|
|
| 795 |
//value = value.replace("«", " ");
|
|
| 796 |
//value = value.replace("»", " ");
|
|
| 797 | 626 |
|
| 798 |
//value = value.replaceAll("[^a-zA-Z0-9 .-_:/!@+=]+", " ");
|
|
| 799 |
|
|
| 800 | 627 |
return value; |
| 801 | 628 |
} else {
|
| 802 | 629 |
return ""; |
| ... | ... | |
| 806 | 633 |
|
| 807 | 634 |
public String cleanId(String value) {
|
| 808 | 635 |
if (value != null) {
|
| 809 |
// TODO DO NOT CHANGE THIS: IT REMOVES ID PREFIX ( 5|datacite____:: |
|
| 810 |
|
|
| 811 |
// to datacite____:: ) |
|
| 812 |
// AND REPLACES OCCURENCES OF DELIM CHARS IN DATA |
|
| 636 |
// DO NOT CHANGE THIS: IT REMOVES ID PREFIX ( "5|datacite____::" to "datacite____::") |
|
| 637 |
// AND REPLACES OCCURRENCES OF DELIM CHARS IN DATA |
|
| 813 | 638 |
value = value.replaceFirst(".*\\|", "");
|
| 814 | 639 |
value = value.replace("\n", "");
|
| 815 | 640 |
value = value.replace(ENCLOSING, ""); |
| ... | ... | |
| 820 | 645 |
} |
| 821 | 646 |
|
| 822 | 647 |
return ENCLOSING + value + ENCLOSING; |
| 823 |
|
|
| 824 | 648 |
} |
| 825 | 649 |
|
| 826 |
|
|
| 827 | 650 |
public String cleanUrl(String value) {
|
| 828 | 651 |
value = value.replace(DELIM, " "); |
| 829 | 652 |
value = value.replace(ENCLOSING, " "); |
| ... | ... | |
| 832 | 655 |
return value; |
| 833 | 656 |
} |
| 834 | 657 |
|
| 835 |
|
|
| 836 | 658 |
public long DATEDIFF(String startDate, String endDate) {
|
| 837 | 659 |
long MILLISECS_PER_DAY = 24 * 60 * 60 * 1000; |
| 838 | 660 |
long days = 0l; |
Also available in: Unified diff
Removed unused and commented code