D-Net

package eu.dnetlib.data.mapreduce.hbase.dataimport;

import com.google.gson.JsonArray;

import com.google.gson.JsonElement;

import com.google.gson.JsonObject;

import com.googlecode.protobuf.format.JsonFormat;

import eu.dnetlib.actionmanager.actions.ActionFactory;

import eu.dnetlib.actionmanager.actions.AtomicAction;

import eu.dnetlib.data.mapreduce.util.StreamUtils;

import eu.dnetlib.data.transform.xml.AbstractDNetXsltFunctions;

import org.apache.avro.data.Json;

import org.apache.commons.lang3.StringUtils;

import org.apache.solr.common.util.StrUtils;

import static eu.dnetlib.data.proto.FieldTypeProtos.*;

import static eu.dnetlib.data.proto.KindProtos.Kind;

import static eu.dnetlib.data.proto.OafProtos.*;

import static eu.dnetlib.data.proto.ResultProtos.Result;

import static eu.dnetlib.data.proto.ResultProtos.Result.*;

import static eu.dnetlib.data.proto.ResultResultProtos.ResultResult;

import static eu.dnetlib.data.proto.TypeProtos.Type;

import static eu.dnetlib.data.mapreduce.hbase.dataimport.ScholixToActions.getQualifier;

import static eu.dnetlib.data.mapreduce.hbase.dataimport.ScholixToActions.getStringValue;

import static eu.dnetlib.data.mapreduce.hbase.dataimport.ScholixToActions.getArrayValues;

import java.util.ArrayList;

import java.util.List;

import java.util.Objects;

import java.util.stream.Collectors;

public class CrossRefToActions {

    public static List<AtomicAction> generateActionsFromDump(final JsonObject rootElement,ActionFactory factory) {

        final List<AtomicAction> actions = new ArrayList<>();

        //Create OAF Proto

        final Oaf.Builder oaf = Oaf.newBuilder();

        //Add Data Info

        oaf.setDataInfo(DataInfo.newBuilder()

                .setInvisible(false)

                .setDeletedbyinference(false)

                .setInferred(false)

                .setTrust("0.9")

                .setProvenanceaction(getQualifier("sysimport:actionset", "dnet:provenanceActions"))

                .build());

        //Adding Kind

        oaf.setKind(Kind.entity);

        //creating Result Proto

        final OafEntity.Builder entity = OafEntity.newBuilder().setType(Type.result);

        //Adding Collected From

        entity.setDateofcollection(Objects.requireNonNull(getStringValue(rootElement, "dateOfCollection")));

        if (rootElement.has("collectedFrom") && rootElement.get("collectedFrom").isJsonArray()){

            StreamUtils.toStream(rootElement.getAsJsonArray("collectedFrom").iterator())

                    .map(JsonElement::getAsJsonObject)

                    .forEach(cf ->

                            {

                             final String id =getStringValue(cf,"id");

                             final String name =getStringValue(cf,"name");

                             if (StringUtils.isNotBlank(id) && StringUtils.isNotBlank(name)) {

                                 final KeyValue collectedFrom = KeyValue.newBuilder()

                                         .setValue(name)

                                         .setKey("10|openaire____::" + AbstractDNetXsltFunctions.md5(StringUtils.substringAfter(id, "::")))

                                         .build();

                                 entity.addCollectedfrom(collectedFrom);

                             }

                            }

                    );

        }

        //Adding identifier

        final String objIdentifier = getStringValue(rootElement, "objIdentifier");

        final String nsPrefix = getStringValue(rootElement,"datasourcePrefix");

        if (StringUtils.isBlank(objIdentifier)) return null;

        final String sourceId = String.format("50|%s::%s", nsPrefix, objIdentifier);

        entity.setId(sourceId);

        final String doi = getStringValue(rootElement, "doi");

        //ADDING PID

        if (doi == null)

            return null;

        entity.addPid(StructuredProperty.newBuilder()

                .setValue(doi)

                .setQualifier(getQualifier("doi", "dnet:pid_types"))

                .build());

        //Create Result Field

        Result.Builder result = Result.newBuilder();

        //Adding Instance

        final String typeValue = getStringValue(rootElement.getAsJsonObject("type"),"value");

        final String cobjValue = getStringValue(rootElement.getAsJsonObject("type"),"cobj");

        //Add UnpayWall instance

        final String best_oa_location_url = getStringValue(rootElement, "best_oa_location_url");

        Instance.Builder instance= Instance.newBuilder();

        instance.setInstancetype(Qualifier.newBuilder()

                .setClassid(cobjValue)

                .setClassname(typeValue)

                .setSchemeid("dnet:publication_resource")

                .setSchemename("dnet:publication_resource")

                .build());

        instance.setHostedby(KeyValue.newBuilder()

                .setKey("10|openaire____::55045bd2a65019fd8e6741a755395c8c")

                .setValue("Unknown Repository")

                .build());

        if (StringUtils.isNotBlank(best_oa_location_url)){

            instance.addUrl(best_oa_location_url);

            instance.setAccessright(Qualifier.newBuilder()

                    .setClassid("OPEN")

                    .setClassname("open access")

                    .setSchemeid("dnet:access_modes")

                    .setSchemename("dnet:access_modes")

                    .build());

            instance.setCollectedfrom(KeyValue.newBuilder()

                    .setValue("UnpayWall")

                    .setKey("10|openaire____::" + AbstractDNetXsltFunctions.md5("unpaywall"))

                    .build());

        } else {

            instance = Instance.newBuilder();

            instance.addUrl(String.format("http://dx.doi.org/%s", doi));

            instance.setAccessright(Qualifier.newBuilder()

                    .setClassid("CLOSED")

                    .setClassname("Closed Access")

                    .setSchemeid("dnet:access_modes")

                    .setSchemename("dnet:access_modes")

                    .build());

            instance.setCollectedfrom(KeyValue.newBuilder()

                    .setValue("CrossRef")

                    .setKey("10|openaire____::" + AbstractDNetXsltFunctions.md5("crossref"))

                    .build());

        }

        result.addInstance(instance.build());

        //Create Metadata Proto

        Metadata.Builder metadata = Metadata.newBuilder();

        //Adding Authors

        final List<Author> authors = createAuthors(rootElement);

        if (authors!= null)

            metadata.addAllAuthor(authors);

        //adding Language

        metadata.setLanguage(Qualifier.newBuilder()

                .setClassid("und")

                .setClassname("Undetermined")

                .setSchemeid("dent:languages")

                .setSchemename("dent:languages")

                .build());

        //Adding subjects

        List<String> subjects =getArrayValues(rootElement, "subject");

        subjects.forEach(s-> metadata.addSubject(StructuredProperty.newBuilder()

                .setValue(s)

                .setQualifier(getQualifier("keyword", "dnet:subject"))

                .build()));

        //Adding titles

        List<String>titles =getArrayValues(rootElement, "title");

        titles.forEach(t->

            metadata.addTitle(StructuredProperty.newBuilder()

                    .setValue(t)

                    .setQualifier(getQualifier("main title", "dnet:dataCite_title"))

                    .build()));

        //Adding date

        String date = getStringValue(rootElement,"issued");

        if (date.length()==4)

            date +="-01-01";

        metadata.setDateofacceptance(StringField.newBuilder().setValue(date).build());

        //Adding description

        String description=null;

        if (rootElement.has("abstract") && rootElement.get("abstract").isJsonArray())

            description =String.join(" ",getArrayValues(rootElement,"abstract"));

        else if (rootElement.has("abstract") )

            description = rootElement.get("abstract").getAsString();

        if(StringUtils.isNotBlank(description))

            metadata.addDescription(StringField.newBuilder().setValue(description).build());

        //Adding Journal

        final String publisher = getStringValue(rootElement,"publisher");

        if (StringUtils.isNotBlank(publisher)){

            final Journal.Builder journal = Journal.newBuilder().setName(publisher);

            if (hasJSONArrayField(rootElement,"issn" )){

                StreamUtils.toStream(rootElement.getAsJsonArray("issn").iterator())

                        .map(JsonElement::getAsJsonObject)

                        .forEach(it -> {

                            final String type = getStringValue(it, "type");

                            final String value = getStringValue(it, "value");

                            if("electronic".equals(type)){

                                journal.setIssnOnline(value);

                            }

                            if ("print".equals(type))

                                journal.setIssnPrinted(value);

                        });

            }

            metadata.setJournal(journal.build());

        }

        metadata.setResulttype(getQualifier(getDefaultResulttype(cobjValue), "dnet:result_typologies"));

        result.setMetadata(metadata.build());

        entity.setResult(result.build());

        oaf.setEntity(entity.build());

        System.out.println(JsonFormat.printToString(oaf.build()));

        return actions;

    }

    private static boolean hasJSONArrayField(final JsonObject root, final String key) {

        return root.has(key) && root.get(key).isJsonArray();

    }

    public static List<Author> createAuthors(final JsonObject root) {

        if (root.has("author") &&  root.get("author").isJsonArray()) {

            final List<Author> authors = new ArrayList<>();

            final JsonArray jsonAuthors = root.getAsJsonArray("author");

            int i = 0;

            for (JsonElement item: jsonAuthors) {

                final JsonObject author = item.getAsJsonObject();

                final Author.Builder result =Author.newBuilder();

                final String given  = getStringValue(author, "given");

                final String family = getStringValue(author, "family");

                final String orchid = getStringValue(author, "ORCID");

                if (StringUtils.isBlank(given) && StringUtils.isBlank(family))

                    continue;

                result.setFullname(given+" "+ family);

                if (StringUtils.isNotBlank(given))

                    result.setName(given);

                if (StringUtils.isNotBlank(family))

                result.setSurname(family);

                if (StringUtils.isNotBlank(orchid))

                {

                    result.addPid(KeyValue.newBuilder()

                            .setValue(orchid)

                            .setKey("ORCID")

                            .build());

                }

                result.setRank(i++);

                authors.add(result.build());

            }

            return authors;

        }

        return null;

    }

    private static String getDefaultResulttype(final String cobjcategory) {

        switch (cobjcategory) {

            case "0029":

                return "software";

            case "0021":

            case "0024":

            case "0025":

            case "0030":

                return "dataset";

            case "0000":

            case "0010":

            case "0018":

            case "0020":

            case "0022":

            case "0023":

            case "0026":

            case "0027":

            case "0028":

            case "0037":

                return "other";

            case "0001":

            case "0002":

            case "0004":

            case "0005":

            case "0006":

            case "0007":

            case "0008":

            case "0009":

            case "0011":

            case "0012":

            case "0013":

            case "0014":

            case "0015":

            case "0016":

            case "0017":

            case "0019":

            case "0031":

            case "0032":

                return "publication";

            default:

                return "publication";

        }

    }

}