D-Net

package eu.dnetlib.data.mapreduce.hbase;

import java.io.Serializable;

/**

 * Created by Alessia Bardi on 2019-04-08.

 *

 * @author Alessia Bardi

 */

public interface Reporter extends Serializable {

	void incrementCounter(String counterGroup, String counterName, long delta);

}

import java.io.ByteArrayOutputStream;

import java.io.IOException;

import java.io.InputStream;

import java.util.*;

import java.util.concurrent.atomic.AtomicInteger;

import java.util.function.Function;

import java.util.stream.Collectors;

import java.util.stream.Stream;

import java.util.zip.Inflater;

import eu.dnetlib.data.mapreduce.hbase.Reporter;

    public final static String doiBoostNSPREFIX = "doiboost____";

    private static Map<String, Pair<String, String>> datasources = new HashMap<String, Pair<String, String>>() {{

        put(MAG.toLowerCase(), new Pair<>("Microsoft Academic Graph", OPENAIRE_PREFIX + SEPARATOR + "microsoft"));

        put(ORCID.toLowerCase(), new Pair<>(ORCID, OPENAIRE_PREFIX + SEPARATOR + "orcid"));

        put(CROSSREF.toLowerCase(), new Pair<>(CROSSREF, OPENAIRE_PREFIX + SEPARATOR + "crossref"));

        put(UNPAYWALL.toLowerCase(), new Pair<>(UNPAYWALL, OPENAIRE_PREFIX + SEPARATOR + "unpaywall"));

    private static String decompressAbstract(final String abstractCompressed) {

            System.out.println("Wrong abstract:" + abstractCompressed);

            throw new RuntimeException(e);

    private static Map<String, FieldTypeProtos.Qualifier> affiliationPIDType = new HashMap<String, FieldTypeProtos.Qualifier>() {{

        put(MAG, FieldTypeProtos.Qualifier.newBuilder().setClassid("mag_id").setClassname("Microsoft Academic Graph Identifier").setSchemename(PID_TYPES)

                .setSchemeid(PID_TYPES).build());

            final String tt = IOUtils.toString(is);

    protected static boolean isValid(final JsonObject rootElement, final Reporter context) {

        final String doi = getStringValue(rootElement, "doi");

        if (doi == null) {

            context.incrementCounter("filtered", "no_doi", 1);

            return false;

        }

        final String type = getStringValue(rootElement, "type");

        if (!typologiesMapping.containsKey(type)) {

            context.incrementCounter("filtered", "unknowntype_" + type, 1);

            return false;

        }

        // fixes #4360 (test publisher)

        final String publisher = getStringValue(rootElement, "publisher");

        if (StringUtils.isNotBlank(publisher) && publisher.equalsIgnoreCase("Test accounts")) {

            context.incrementCounter("filtered", "test_publisher", 1);

            return false;

        }

        List<JsonObject> authors = getArrayObjects(rootElement, "authors");

        boolean hasAuthors = false;

        for (JsonObject author : authors) {

            final String given = getStringValue(author, "given");

            final String family = getStringValue(author, "family");

            String fullname = getStringValue(author, "fullname");

            if (StringUtils.isBlank(fullname) && StringUtils.isNotBlank(given) && StringUtils.isNotBlank(family)) {

                fullname = String.format("%s %s", given, family);

            }

            // fixes #4368

            if (fullname.equalsIgnoreCase("Addie Jackson") && publisher.equalsIgnoreCase("Elsevier BV")) {

                context.incrementCounter("invalid_author", "addiejackson", 1);

                context.incrementCounter("filtered", "invalid_authors", 1);

                return false;

            }

            if (isValidAuthorName(fullname, context)) hasAuthors = true;

        }

        if (!hasAuthors) {

            context.incrementCounter("filtered", "invalid_authors", 1);

            return false;

        }

        // fixes #4360

        if (getCleanedTitles(rootElement).isEmpty()) {

            context.incrementCounter("filtered", "invalid_title", 1);

            return false;

        }

        return true;

    }

    private static List<String> getCleanedTitles(final JsonObject rootElement) {

        List<String> titles = getArrayValues(rootElement, "title");

        return titles.stream().filter(t -> StringUtils.isNotBlank(t) && !t.equalsIgnoreCase("[NO TITLE AVAILABLE]")).collect(Collectors.toList());

    }

    private static boolean isValidAuthorName(final String fullName, final Reporter context) {

        if (StringUtils.isBlank(fullName)) {

            if(context != null) context.incrementCounter("invalid_author", "blank", 1);

            return false;

        }

        // fixes #4391 and subtasks related to DOIBoost

        switch (fullName) {

        case ",":

        case "none none":

        case "none &na;":

        case "(:null)":

        case "&na; &na;": {

            if(context != null) context.incrementCounter("invalid_author", "value_" + fullName, 1);

            return false;

        }

        }

        return true;

    }

    public static List<AtomicAction> generatePublicationActionsFromDump(final JsonObject rootElement,

            final ActionFactory factory,

            final String setName,

            final Agent agent,

            boolean invisible,

            final boolean onlyOrganization,

            final Reporter context) {

        if (!isValid(rootElement, context)) return null;

        entity.setDateofcollection("2019-02-15");

        if (rootElement.has("collectedFrom") && rootElement.get("collectedFrom").isJsonArray()) {

                                final String id = datasources.get(cf.toLowerCase()).getValue();

                                final String name = datasources.get(cf.toLowerCase()).getKey();

        entity.addOriginalId(doi);

        final String type = getStringValue(rootElement, "type");

        // TODO: workaround for #4362: remove it when UnpayWall is correctly mapped

        List<JsonObject> unpaywallLicenses = getArrayObjects(rootElement, "license").stream().filter(prov -> {

            String provS = getStringValue(prov, "provenance");

            if (StringUtils.isNotBlank(provS) && provS.equalsIgnoreCase(UNPAYWALL)) return true;

            else return false;

        }).collect(Collectors.toList());

        Stream.concat(unpaywallLicenses.stream(), getArrayObjects(rootElement, "instances").stream()).map(it ->

            ResultProtos.Result.Instance.Builder instance = ResultProtos.Result.Instance.newBuilder();

            final String acc_class_id = it.get("access-rights").getAsString();

            switch (acc_class_id) {

            case "OPEN": {

                acc_class_value = "open access";

                break;

            }

            case "CLOSED": {

                acc_class_value = "closed access";

                break;

            }

            default: {

                acc_class_value = "not available";

            }

            final String id = datasources.get(it.get("provenance").getAsString().toLowerCase()).getValue();

            final String name = datasources.get(it.get("provenance").getAsString().toLowerCase()).getKey();

            return instance.build();

        JsonObject hostedByOpenAire = null;

        if (rootElement.has("hostedByOpenAire")) {

            hostedByOpenAire = rootElement.getAsJsonObject("hostedByOpenAire");

        }

            instance.setInstancetype(FieldTypeProtos.Qualifier.newBuilder()

                    .setClassid(cobjValue)

                    .setClassname(typeValue)

                    .setSchemeid("dnet:publication_resource")

                    .setSchemename("dnet:publication_resource")

                    .build());

                    .setValue(CROSSREF)

            if (hostedByOpenAire == null)

                instance.setHostedby(FieldTypeProtos.KeyValue.newBuilder()

                        .setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + "55045bd2a65019fd8e6741a755395c8c")

                        .setValue("Unknown Repository")

                        .build());

            else {

                instance.setHostedby(FieldTypeProtos.KeyValue.newBuilder()

                        .setKey(AbstractDNetXsltFunctions.oafSplitId("datasource", hostedByOpenAire.get("id").getAsString()))

                        .setValue(hostedByOpenAire.get("name").getAsString())

                        .build());

            }

        } else {

            //Should never enter here becasue of the isValid method at the beginning.

            context.incrementCounter("filtered", "unexpected_no_authors", 1);

        List<String> subjects = getArrayValues(rootElement, "subject");

        subjects.forEach(s -> metadata.addSubject(FieldTypeProtos.StructuredProperty.newBuilder()

        List<String> titles = getCleanedTitles(rootElement);

        titles.forEach(t ->

        );

        final String publisher = getStringValue(rootElement, "publisher");

        if (StringUtils.isNotBlank(publisher)) {

            if (hasJSONArrayField(rootElement, "issn")) {

                            if ("electronic".equals(issntype)) {

        //System.out.println(JsonFormat.printToString(oaf.build()));

                if (gridOrganization != null) {

                    actionList.add(factory

                            .createAtomicAction(setName, agent, o.getEntity().getId(), "organizationOrganization_dedupSimilarity_isSimilarTo", gridOrganization,

                                    "".getBytes()));

                    actionList.add(factory

                            .createAtomicAction(setName, agent, gridOrganization, "organizationOrganization_dedupSimilarity_isSimilarTo", o.getEntity().getId(),

                                    "".getBytes()));

        if (pidList != null) {

            for (FieldTypeProtos.StructuredProperty p : pidList) {

                if (p.getQualifier().getClassname().equals("grid")) {

                    return "20|grid________" + SEPARATOR + AbstractDNetXsltFunctions.md5(p.getValue());

    private static List<AtomicAction> createPublicationOrganizationRelation(final OafProtos.Oaf publication,

            final OafProtos.Oaf organization,

            final ActionFactory factory,

            final String setName,

            final Agent agent) {

                .setValue(datasources.get(MAG.toLowerCase()).getKey())

                .setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + AbstractDNetXsltFunctions

                        .md5(StringUtils.substringAfter(datasources.get(MAG.toLowerCase()).getValue(), SEPARATOR)))

        result.add(factory.createAtomicAction(setName, agent, publication.getEntity().getId(), "resultOrganization_affiliation_hasAuthorInstitution",

                organization.getEntity().getId(), roaf.build().toByteArray()));

        result.add(factory.createAtomicAction(setName, agent, organization.getEntity().getId(), "resultOrganization_affiliation_isAuthorInstitutionOf",

                publication.getEntity().getId(), roaf.build().toByteArray()));

    private static void settingRelevantDate(JsonObject rootElement,

            ResultProtos.Result.Metadata.Builder metadata,

            final String jsonKey,

            final String dictionaryKey,

            final boolean addToDateOfAcceptance) {

        String date = getStringValue(rootElement, jsonKey);

                    .setQualifier(getQualifier(dictionaryKey, "dnet:dataCite_date"))

        if (StringUtils.contains(value, "orcid.org")) {

            return pid.setValue(value.replaceAll("https://orcid.org/", ""))

        if (StringUtils.contains(value, "academic.microsoft.com/#/detail")) {

            return pid.setValue(value.replaceAll("https://academic.microsoft.com/#/detail/", ""))

            } else

                affiliationIdentifiers.put(it.get("value").getAsString(), affiliationPIDType.get(it.get("schema").getAsString()));

            entity.setId("20|microsoft___" + SEPARATOR + AbstractDNetXsltFunctions.md5(microsoftID));

            final String id = datasources.get(affiliation.get("provenance").getAsString().toLowerCase()).getValue();

            final String name = datasources.get(affiliation.get("provenance").getAsString().toLowerCase()).getKey();

        return null;

    public static Pair<List<FieldTypeProtos.Author>, Collection<OafProtos.Oaf>> createAuthorsOrganization(final JsonObject root) {

        final AtomicInteger counter = new AtomicInteger(1);

            if (!isValidAuthorName(fullname, null)) {

                            (a, b) -> a

        return new Pair<>(collect, affiliations.values());

import java.io.IOException;

import java.util.List;

import eu.dnetlib.data.mapreduce.hbase.Reporter;

            List<AtomicAction> atomicActions =

                    DOIBoostToActions.generatePublicationActionsFromDump(rootElement, factory, setName, agent, invisible, onlyOrganization,

                            (Reporter) (counterGroup, counterName, delta) -> context.getCounter(counterGroup, counterName).increment(delta));

            if (atomicActions != null) {

                for (AtomicAction action : atomicActions) {