D-Net

    @Test

    public void testMultipleDOIBoostActionDiscardMany() throws IOException {

        doTestAllDOIBoostAction("/eu/dnetlib/data/mapreduce/actions/doiboost_discard_many.json");

    }

{"publisher": "American Medical Association (AMA)", "doi": "10.1001/2013.jamadermatol.238", "license": [], "title": ["Comparison of Ustekinumab With Other Biological Agents for the Treatment of Moderate to Severe Plaque Psoriasis"], "issued": "2012-12-1", "abstract": [{"provenance": "MAG", "value": "eJylVU2P2zYQ/StzKRADsmrLWq/dnNJkA2yRdIPsAj0EPVDS2GIjkS5J2ev++r4hbVfGpkCKXmyD\n5My8j5nxQ/UH10HvmZ4s1bbfKccUWibebHSt6iPZDQ0+8Fdthl5VdNChxQMV5MLipaNK285u8boj\ntWUTPAK02cY0n7x1Wnnt6Y1jRco09Mh7djoc6d40/EyvPr15vJ+Q6i1CVDN0gXYq6JgnFuttw04F\nwLLkJZZp16k/B3ydk+f0TgVFj3ZwNXv6jUHFNEMduCFF/gj8PXLWiFeubgW6gNsBh23oF2UG5Y60\nmGc0X6+LTCq958rFUxwWszkOE6mPd+8+3P96B9hD9ZGbSUZ3faU8ZzHjW1u3ThmmD7pyiM4i47rT\nRuQJQNv5fGv3OT2GoTlCi07kt0Ywa1N3QwPIyNDYXv+Fn+ARnO06/EzRgv2l4CfnmqQY5Km5smTd\n9zgE+VF6JHprqVXNf9D97jk4lWgkWo2cHiSC0xWQabi9Y3yY0B2pOlKRWitLmBvt1dYx97gnx952\newThGRTwbPxwLjbOeNJLaHjIdfY1KpXRBrrZw3TYnYzOhLyIu40vz4Qzqc2wkJoBfMEio8qChR/c\nBkKSks7VZi+QBB6u8RqeJvGyJCEw7wRqsnzneK/t4IGFVYikIs0XTpxb92iAPM4J/ayO+KUMGQ4H\n675Sz0FNlVHdUR4clBdCG+v6pNBGhyBuLoABEnovRoh7nf8J7b/Rz9xMMc/oNJ/OgfHUZC/OTfMv\nd6dhFCijMrWNi0InKk8tfztW1a3m/cmqin0Q0DDIyYEoDw1yLASYot0Bdpw6WntIml0toAGXIoHy\n3tZahXPT+wDvfBBp0V9eb42WDRa7rdVbGQPbND4WTXBEM0W3Nz9AN1kWwgeDcBmK66FSjep0f4Hw\nKiaL/UJfHj7/jtWRr8rXtEa6GkG66mSiA7s9vP7y1t3HJ7P5dJHflFgbqmPprl1I6R4+43qWL1av\nCW8zWuRlOS1u82UxSa5AeMPuOqDIZ7epZIyZ52WB9LPlhKohfFMlDMRZiGt62mw6/awv9CQ94CxH\n6Wf5vJzO8lUxoehVHDXoqnY8yHKtO1QbGZdWYtSg4wH+TefFj8UCJ62udIAPsmbkTfQHTfG/DDpv\nvQh9uc7L1Qj7Ypmv1tM5vsrl5Lwa0viEoUdFw7WzMl4wRaBdQPqUsCzyohglhDXrcooyi3IS/RFU\nT9Oau25EMIbe5MvFlUuLm2lR5kU5yfGHIUssjtL7NA2jjYEl9R1L+Ho8eoXlyohzl//wtIdaLJR/\nejgb9dNp6i/9KL1jbBi1RP43lFHR7g==\n"}], "issn": [{"type": "print", "value": "0003-987X"}], "doi-url": "http://dx.doi.org/10.1001/2013.jamadermatol.238", "instances": [{"url": "http://jamanetwork.com/journals/jamadermatology/fullarticle/1377948", "provenance": "Crossref", "access-rights": "UNKNOWN"}], "authors": [{"affiliations": [{"official-page": "http://www.washington.edu/", "provenance": "MAG", "value": "University of Washington", "identifiers": [{"value": "http://en.wikipedia.org/wiki/University_of_Washington", "schema": "wikpedia"}, {"value": "grid.34477.33", "schema": "grid.ac"}, {"value": "https://academic.microsoft.com/#/detail/201448701", "schema": "URL"}]}], "given": "Vincent W.", "identifiers": [{"provenance": "MAG", "value": "https://academic.microsoft.com/#/detail/2116346541", "schema": "URL"}], "fullname": "Vincent W. Lin", "family": "Lin"}, {"affiliations": [{"official-page": "http://www.childrenshospital.org/", "provenance": "MAG", "value": "Boston Children's Hospital", "identifiers": [{"value": "http://en.wikipedia.org/wiki/Boston_Children's_Hospital", "schema": "wikpedia"}, {"value": "grid.2515.3", "schema": "grid.ac"}, {"value": "https://academic.microsoft.com/#/detail/1288882113", "schema": "URL"}]}], "given": "Sarah", "identifiers": [{"provenance": "MAG", "value": "https://academic.microsoft.com/#/detail/2011423229", "schema": "URL"}], "fullname": "Sarah Ringold", "family": "Ringold"}, {"affiliations": [{"official-page": "http://www.washington.edu/", "provenance": "MAG", "value": "University of Washington", "identifiers": [{"value": "http://en.wikipedia.org/wiki/University_of_Washington", "schema": "wikpedia"}, {"value": "grid.34477.33", "schema": "grid.ac"}, {"value": "https://academic.microsoft.com/#/detail/201448701", "schema": "URL"}]}], "given": "Emily Beth", "identifiers": [{"provenance": "MAG", "value": "https://academic.microsoft.com/#/detail/1950057684", "schema": "URL"}], "fullname": "Emily Beth Devine", "family": "Devine"}], "published-print": "2012-12-1", "collectedFrom": ["Crossref", "MAG"], "accepted": null, "type": "journal-article", "published-online": null, "subject": ["Dermatology", "General Medicine"]}

{"publisher": "American Medical Association (AMA)", "doi": "10.1001/2013.jamainternmed.335", "license": [], "title": [""], "issued": "2012-11-26", "abstract": [], "issn": [{"type": "print", "value": "0003-9926"}], "doi-url": "http://dx.doi.org/10.1001/2013.jamainternmed.335", "instances": [{"url": "http://jamanetwork.com/journals/jamainternalmedicine/fullarticle/1384244", "provenance": "Crossref", "access-rights": "UNKNOWN"}], "authors": [{"affiliations": [], "given": "Terrence", "identifiers": [{"provenance": "MAG", "value": "https://academic.microsoft.com/#/detail/2686390077", "schema": "URL"}], "fullname": "Terrence Shaneyfelt", "family": "Shaneyfelt"}], "published-print": "2012-11-26", "collectedFrom": ["Crossref", "MAG"], "accepted": null, "type": "journal-article", "published-online": null, "subject": ["Internal Medicine"]}

{"publisher": "American Medical Association (AMA)", "doi": "10.1001/2013.jamainternmed.82", "license": [], "title": ["[NO TITLE AVAILABLE"], "issued": "2012-12-10", "abstract": [], "issn": [{"type": "print", "value": "0003-9926"}], "doi-url": "http://dx.doi.org/10.1001/2013.jamainternmed.82", "instances": [{"url": "http://jamanetwork.com/journals/jamainternalmedicine/fullarticle/1389243", "provenance": "Crossref", "access-rights": "UNKNOWN"}], "authors": [{"affiliations": [{"official-page": "http://www.upenn.edu/", "provenance": "MAG", "value": "University of Pennsylvania", "identifiers": [{"value": "http://en.wikipedia.org/wiki/University_of_Pennsylvania", "schema": "wikpedia"}, {"value": "grid.25879.31", "schema": "grid.ac"}, {"value": "https://academic.microsoft.com/#/detail/79576946", "schema": "URL"}]}], "given": "Shreya", "identifiers": [{"provenance": "MAG", "value": "https://academic.microsoft.com/#/detail/2092315005", "schema": "URL"}], "fullname": "Shreya Kangovi", "family": "Kangovi"}, {"affiliations": [{"official-page": "http://www.upenn.edu/", "provenance": "MAG", "value": "University of Pennsylvania", "identifiers": [{"value": "http://en.wikipedia.org/wiki/University_of_Pennsylvania", "schema": "wikpedia"}, {"value": "grid.25879.31", "schema": "grid.ac"}, {"value": "https://academic.microsoft.com/#/detail/79576946", "schema": "URL"}]}], "given": "Judith A.", "identifiers": [{"provenance": "MAG", "value": "https://academic.microsoft.com/#/detail/2012187813", "schema": "URL"}], "fullname": "Judith A. Long", "family": "Long"}, {"affiliations": [{"official-page": "http://www.upenn.edu/", "provenance": "MAG", "value": "University of Pennsylvania", "identifiers": [{"value": "http://en.wikipedia.org/wiki/University_of_Pennsylvania", "schema": "wikpedia"}, {"value": "grid.25879.31", "schema": "grid.ac"}, {"value": "https://academic.microsoft.com/#/detail/79576946", "schema": "URL"}]}], "given": "Ezekiel", "identifiers": [{"provenance": "MAG", "value": "https://academic.microsoft.com/#/detail/47466971", "schema": "URL"}], "fullname": "Ezekiel Emanuel", "family": "Emanuel"}], "published-print": "2012-12-10", "collectedFrom": ["Crossref", "MAG"], "accepted": null, "type": "journal-article", "published-online": null, "subject": ["Internal Medicine"]}

{"publisher": "American Medical Association (AMA)", "doi": "10.1001/archderm.100.4.515", "license": [], "title": ["Tuberculoid leprosy in state of reaction"], "issued": "1969-10-1", "abstract": [], "issn": [{"type": "print", "value": "0003-987X"}], "doi-url": "http://dx.doi.org/10.1001/archderm.100.4.515", "instances": [], "authors": [{"affiliations": [], "given": "O.", "identifiers": [{"provenance": "MAG", "value": "https://academic.microsoft.com/#/detail/2639594543", "schema": "URL"}], "fullname": "O. Canizares", "family": "Canizares"}], "published-print": "1969-10-1", "collectedFrom": ["Crossref", "MAG"], "accepted": null, "type": "journal-article", "published-online": null, "subject": ["Dermatology", "General Medicine"]}

{"publisher": "American Medical Association (AMA)", "doi": "10.1001/archderm.101.6.679", "license": [], "title": ["Multiple palmar basal cell epitheliomas in basal cell nevus syndrome"], "issued": "1970-6-1", "abstract": [], "issn": [{"type": "print", "value": "0003-987X"}], "doi-url": "http://dx.doi.org/10.1001/archderm.101.6.679", "instances": [], "authors": [{"affiliations": null, "given": "K.", "identifiers": null, "fullname": "K. Holubar", "family": "Holubar"}], "published-print": "1970-6-1", "collectedFrom": ["Crossref"], "accepted": null, "type": "journal-article", "published-online": null, "subject": ["Dermatology", "General Medicine"]}

{"publisher": "American Medical Association (AMA)", "doi": "10.1001/archderm.103.5.494", "license": [], "title": ["Syringoma of the vulva"], "issued": "1971-5-1", "abstract": [], "issn": [{"type": "print", "value": "0003-987X"}], "doi-url": "http://dx.doi.org/10.1001/archderm.103.5.494", "instances": [], "authors": [{"affiliations": null, "given": "S. J.", "identifiers": null, "fullname": "S. J. Carneiro", "family": "Carneiro"}], "published-print": "1971-5-1", "collectedFrom": ["Crossref"], "accepted": null, "type": "journal-article", "published-online": null, "subject": ["Dermatology", "General Medicine"]}

{"publisher": "American Medical Association (AMA)", "doi": "10.1001/archderm.107.6.872", "license": [], "title": ["Atypical gingivostomatitis. Nineteen cases"], "issued": "1973-6-1", "abstract": [], "issn": [{"type": "print", "value": "0003-987X"}], "doi-url": "http://dx.doi.org/10.1001/archderm.107.6.872", "instances": [], "authors": [{"affiliations": null, "given": "none", "identifiers": null, "fullname": "", "family": "none"}], "published-print": "1973-6-1", "collectedFrom": ["Crossref"], "accepted": null, "type": "journal-article", "published-online": null, "subject": ["Dermatology", "General Medicine"]}

{"publisher": "American Medical Association (AMA)", "doi": "10.1001/archderm.108.2.215", "license": [], "title": ["Instantaneous decompression and the skin"], "issued": "1973-8-1", "abstract": [], "issn": [{"type": "print", "value": "0003-987X"}], "doi-url": "http://dx.doi.org/10.1001/archderm.108.2.215", "instances": [], "authors": [{"affiliations": null, "given": "", "identifiers": null, "fullname": "", "family": ""}], "published-print": "1973-8-1", "collectedFrom": ["Crossref"], "accepted": null, "type": "journal-article", "published-online": null, "subject": ["Dermatology", "General Medicine"]}

{"publisher": "American Medical Association (AMA)", "doi": "10.1001/archderm.108.2.215", "license": [], "title": ["Instantaneous decompression and the skin"], "issued": "1973-8-1", "abstract": [], "issn": [{"type": "print", "value": "0003-987X"}], "doi-url": "http://dx.doi.org/10.1001/archderm.108.2.215", "instances": [], "authors": [{"affiliations": null, "given": "", "identifiers": null, "fullname": "", "family": ""}, {"affiliations": [], "given": "O.", "identifiers": [{"provenance": "MAG", "value": "https://academic.microsoft.com/#/detail/2639594543", "schema": "URL"}], "fullname": "O. Canizares", "family": "Canizares"}], "published-print": "1973-8-1", "collectedFrom": ["Crossref"], "accepted": null, "type": "journal-article", "published-online": null, "subject": ["Dermatology", "General Medicine"]}

	public static final String MAG = "MAG";

	public static final String ORCID = "ORCID";

	public static final String CROSSREF = "Crossref";

	public static final String UNPAYWALL = "UnpayWall";

	public static final String GRID_AC = "grid.ac";

	public static final String WIKPEDIA = "wikpedia";

	public final static String doiBoostNSPREFIX = "doiboost____";

	public static final String OPENAIRE_PREFIX = "openaire____";

	public static final String SEPARATOR = "::";

	private static Map<String, Pair<String, String>> datasources = new HashMap<String, Pair<String, String>>() {{

		put(MAG.toLowerCase(), new Pair<>("Microsoft Academic Graph", OPENAIRE_PREFIX + SEPARATOR + "microsoft"));

		put(ORCID.toLowerCase(), new Pair<>(ORCID, OPENAIRE_PREFIX + SEPARATOR + "orcid"));

		put(CROSSREF.toLowerCase(), new Pair<>(CROSSREF, OPENAIRE_PREFIX + SEPARATOR + "crossref"));

		put(UNPAYWALL.toLowerCase(), new Pair<>(UNPAYWALL, OPENAIRE_PREFIX + SEPARATOR + "unpaywall"));

	}};

	private static String decompressAbstract(final String abstractCompressed) {

		try {

			byte[] byteArray = Base64.decodeBase64(abstractCompressed.getBytes());

			final Inflater decompresser = new Inflater();

			decompresser.setInput(byteArray);

			final ByteArrayOutputStream bos = new ByteArrayOutputStream(byteArray.length);

			byte[] buffer = new byte[8192];

			while (!decompresser.finished()) {

				int size = decompresser.inflate(buffer);

				bos.write(buffer, 0, size);

			}

			byte[] unzippeddata = bos.toByteArray();

			decompresser.end();

			return new String(unzippeddata);

		} catch (Throwable e) {

			System.out.println("Wrong abstract:" + abstractCompressed);

			throw new RuntimeException(e);

		}

	}

	public static final String PID_TYPES = "dnet:pid_types";

	private static Map<String, FieldTypeProtos.Qualifier> affiliationPIDType = new HashMap<String, FieldTypeProtos.Qualifier>() {{

		put(MAG, FieldTypeProtos.Qualifier.newBuilder().setClassid("mag_id").setClassname("Microsoft Academic Graph Identifier").setSchemename(PID_TYPES)

				.setSchemeid(PID_TYPES).build());

		put(GRID_AC, getQualifier("grid", PID_TYPES));

		put(WIKPEDIA, getQualifier("urn", PID_TYPES));

	}};

	static Map<String, Map<String, String>> typologiesMapping;

	static {

		try {

			final InputStream is = DOIBoostToActions.class.getResourceAsStream("/eu/dnetlib/data/mapreduce/hbase/dataimport/mapping_typologies.json");

			final String tt = IOUtils.toString(is);

			typologiesMapping = new Gson().fromJson(tt, Map.class);

		} catch (IOException e) {

			e.printStackTrace();

		}

	}

	protected static boolean isValid(final JsonObject rootElement, final Reporter context) {

		final String doi = getStringValue(rootElement, "doi");

		if (doi == null) {

			context.incrementCounter("filtered", "no_doi", 1);

			return false;

		}

		final String type = getStringValue(rootElement, "type");

		if (!typologiesMapping.containsKey(type)) {

			context.incrementCounter("filtered", "unknowntype_" + type, 1);

			return false;

		}

		// fixes #4360 (test publisher)

		final String publisher = getStringValue(rootElement, "publisher");

		if (publisher.equalsIgnoreCase("Test accounts")) {

			context.incrementCounter("filtered", "test_publisher", 1);

			return false;

		}

		List<JsonObject> authors = getArrayObjects(rootElement, "authors");

		boolean hasAuthors = false;

		for (JsonObject author : authors) {

			final String given = getStringValue(author, "given");

			final String family = getStringValue(author, "family");

			String fullname = getStringValue(author, "fullname");

			if (StringUtils.isBlank(fullname) && StringUtils.isNotBlank(given) && StringUtils.isNotBlank(family)) {

				fullname = String.format("%s %s", given, family);

			}

			// fixes #4368

			if (fullname.equalsIgnoreCase("Addie Jackson") && publisher.equalsIgnoreCase("Elsevier BV")) {

				context.incrementCounter("invalid_author", "addiejackson", 1);

				context.incrementCounter("filtered", "invalid_authors", 1);

				return false;

			}

			if (isValidAuthorName(fullname, context)) hasAuthors = true;

		}

		if (!hasAuthors) {

			context.incrementCounter("filtered", "invalid_authors", 1);

			return false;

		}

		// fixes #4360

		if (getCleanedTitles(rootElement).isEmpty()) {

			context.incrementCounter("filtered", "invalid_title", 1);

			return false;

		}

		return true;

	}

	private static List<String> getCleanedTitles(final JsonObject rootElement) {

		List<String> titles = getArrayValues(rootElement, "title");

		return titles.stream().filter(t -> StringUtils.isNotBlank(t) && !t.equalsIgnoreCase("[NO TITLE AVAILABLE]")).collect(Collectors.toList());

	}

	private static boolean isValidAuthorName(final String fullName, final Reporter context) {

		if (StringUtils.isBlank(fullName)) {

			if(context != null) context.incrementCounter("invalid_author", "blank", 1);

			return false;

		}

		// fixes #4391 and subtasks related to DOIBoost

		switch (fullName) {

		case ",":

		case "none none":

		case "none &na;":

		case "(:null)":

		case "&na; &na;": {

			if(context != null) context.incrementCounter("invalid_author", "value_" + fullName, 1);

			return false;

			}

		}

		return true;

	}

	public static List<AtomicAction> generatePublicationActionsFromDump(final JsonObject rootElement,

			final ActionFactory factory,

			final String setName,

			final Agent agent,

			boolean invisible,

			final boolean onlyOrganization,

			final Reporter context) {

		if (!isValid(rootElement, context)) return null;

		//Create OAF Proto

		final OafProtos.Oaf.Builder oaf = OafProtos.Oaf.newBuilder();

		//Add Data Info

		oaf.setDataInfo(FieldTypeProtos.DataInfo.newBuilder()

				.setInvisible(invisible)

				.setDeletedbyinference(false)

				.setInferred(false)

				.setTrust("0.9")

				.setProvenanceaction(getQualifier("sysimport:actionset", "dnet:provenanceActions"))

				.build());

		//Adding Kind

		oaf.setKind(KindProtos.Kind.entity);

		//creating Result Proto

		final OafProtos.OafEntity.Builder entity = OafProtos.OafEntity.newBuilder().setType(TypeProtos.Type.result);

		entity.setDateofcollection("2019-02-15");

		if (rootElement.has("collectedFrom") && rootElement.get("collectedFrom").isJsonArray()) {

			StreamUtils.toStream(rootElement.getAsJsonArray("collectedFrom").iterator())

					.map(JsonElement::getAsString)

					.forEach(cf -> {

								final String id = datasources.get(cf.toLowerCase()).getValue();

								final String name = datasources.get(cf.toLowerCase()).getKey();

								if (StringUtils.isNotBlank(id) && StringUtils.isNotBlank(name)) {

									final FieldTypeProtos.KeyValue collectedFrom = FieldTypeProtos.KeyValue.newBuilder()

											.setValue(name)

											.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + AbstractDNetXsltFunctions.md5(StringUtils.substringAfter(id, SEPARATOR)))

											.build();

									entity.addCollectedfrom(collectedFrom);

								}

							}

					);

		}

		//Adding identifier

		final String doi = getStringValue(rootElement, "doi");

		entity.addOriginalId(doi);

		final String sourceId = String.format("50|%s" + SEPARATOR + "%s", doiBoostNSPREFIX, AbstractDNetXsltFunctions.md5(doi));

		entity.setId(sourceId);

		entity.addPid(FieldTypeProtos.StructuredProperty.newBuilder()

				.setValue(doi)

				.setQualifier(getQualifier("doi", PID_TYPES))

				.build());

		//Create Result Field

		ResultProtos.Result.Builder result = ResultProtos.Result.newBuilder();

		final String type = getStringValue(rootElement, "type");

		//Adding Instances

		final String typeValue = typologiesMapping.get(type).get("value");

		final String cobjValue = typologiesMapping.get(type).get("cobj");

		// TODO: workaround for #4362: remove it when UnpayWall is correctly mapped

		List<JsonObject> unpaywallLicenses = getArrayObjects(rootElement, "license").stream().filter(prov -> {

			String provS = getStringValue(prov, "provenance");

			if (StringUtils.isNotBlank(provS) && provS.equalsIgnoreCase(UNPAYWALL)) return true;

			else return false;

		}).collect(Collectors.toList());

		Stream.concat(unpaywallLicenses.stream(), getArrayObjects(rootElement, "instances").stream()).map(it ->

		{

			ResultProtos.Result.Instance.Builder instance = ResultProtos.Result.Instance.newBuilder();

			instance.setInstancetype(FieldTypeProtos.Qualifier.newBuilder()

					.setClassid(cobjValue)

					.setClassname(typeValue)

					.setSchemeid("dnet:publication_resource")

					.setSchemename("dnet:publication_resource")

					.build());

			instance.setHostedby(FieldTypeProtos.KeyValue.newBuilder()

					.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + "55045bd2a65019fd8e6741a755395c8c")

					.setValue("Unknown Repository")

					.build());

			final String acc_class_id = it.get("access-rights").getAsString();

			String acc_class_value;

			switch (acc_class_id) {

			case "OPEN": {

				acc_class_value = "open access";

				break;

			}

			case "CLOSED": {

				acc_class_value = "closed access";

				break;

			}

			default: {

				acc_class_value = "not available";

			}

			}

			instance.addUrl(it.get("url").getAsString());

			instance.setAccessright(FieldTypeProtos.Qualifier.newBuilder()

					.setClassid(acc_class_id)

					.setClassname(acc_class_value)

					.setSchemeid("dnet:access_modes")

					.setSchemename("dnet:access_modes")

					.build());

			final String id = datasources.get(it.get("provenance").getAsString().toLowerCase()).getValue();

			final String name = datasources.get(it.get("provenance").getAsString().toLowerCase()).getKey();

			if (StringUtils.isNotBlank(id) && StringUtils.isNotBlank(name)) {

				final FieldTypeProtos.KeyValue collectedFrom = FieldTypeProtos.KeyValue.newBuilder()

						.setValue(name)

						.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + AbstractDNetXsltFunctions.md5(StringUtils.substringAfter(id, SEPARATOR)))

						.build();

				instance.setCollectedfrom(collectedFrom);

			}

			return instance.build();

		}).forEach(result::addInstance);

		//Adding DOI URL as  Instance

		final String doiURL = getStringValue(rootElement, "doi-url");

		JsonObject hostedByOpenAire = null;

		if (rootElement.has("hostedByOpenAire")) {

			hostedByOpenAire = rootElement.getAsJsonObject("hostedByOpenAire");

		}

		if (StringUtils.isNotBlank(doiURL)) {

			final ResultProtos.Result.Instance.Builder instance = ResultProtos.Result.Instance.newBuilder();

			instance.addUrl(doiURL);

			instance.setInstancetype(FieldTypeProtos.Qualifier.newBuilder()

					.setClassid(cobjValue)

					.setClassname(typeValue)

					.setSchemeid("dnet:publication_resource")

					.setSchemename("dnet:publication_resource")

					.build());

			instance.setAccessright(FieldTypeProtos.Qualifier.newBuilder()

					.setClassid("CLOSED")

					.setClassname("Closed Access")

					.setSchemeid("dnet:access_modes")

					.setSchemename("dnet:access_modes")

					.build());

			instance.setCollectedfrom(FieldTypeProtos.KeyValue.newBuilder()

					.setValue(CROSSREF)

					.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + AbstractDNetXsltFunctions.md5("crossref"))

					.build());

			if (hostedByOpenAire == null)

				instance.setHostedby(FieldTypeProtos.KeyValue.newBuilder()

						.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + "55045bd2a65019fd8e6741a755395c8c")

						.setValue("Unknown Repository")

						.build());

			else {

				instance.setHostedby(FieldTypeProtos.KeyValue.newBuilder()

						.setKey(AbstractDNetXsltFunctions.oafSplitId("datasource", hostedByOpenAire.get("id").getAsString()))

						.setValue(hostedByOpenAire.get("name").getAsString())

						.build());

			}

			result.addInstance(instance);

		}

		//Create Metadata Proto

		final ResultProtos.Result.Metadata.Builder metadata = ResultProtos.Result.Metadata.newBuilder();

		Pair<List<FieldTypeProtos.Author>, Collection<OafProtos.Oaf>> authorsOrganizations = createAuthorsOrganization(rootElement);

		if (authorsOrganizations.getKey().size() > 0) {

			metadata.addAllAuthor(authorsOrganizations.getKey());

		} else {

			//Should never enter here becasue of the isValid method at the beginning.

			context.incrementCounter("filtered", "unexpected_no_authors", 1);

			return null;

		}

		//adding Language

		metadata.setLanguage(FieldTypeProtos.Qualifier.newBuilder()

				.setClassid("und")

				.setClassname("Undetermined")

				.setSchemeid("dent:languages")

				.setSchemename("dent:languages")

				.build());

		//Adding subjects

		List<String> subjects = getArrayValues(rootElement, "subject");

		subjects.forEach(s -> metadata.addSubject(FieldTypeProtos.StructuredProperty.newBuilder()

				.setValue(s)

				.setQualifier(getQualifier("keyword", "dnet:subject"))

				.build()));

		List<String> titles = getCleanedTitles(rootElement);

		titles.forEach(t ->

				metadata.addTitle(FieldTypeProtos.StructuredProperty.newBuilder()

						.setValue(t)

						.setQualifier(getQualifier("main title", "dnet:dataCite_title"))

						.build()));

		settingRelevantDate(rootElement, metadata, "issued", "issued", true);

		settingRelevantDate(rootElement, metadata, "accepted", "accepted", false);

		settingRelevantDate(rootElement, metadata, "published-online", "published-online", false);

		settingRelevantDate(rootElement, metadata, "published-print", "published-print", false);

		getArrayObjects(rootElement, "abstract").forEach(d ->

				{

					if (MAG.equals(d.get("provenance").getAsString()))

						metadata.addDescription(FieldTypeProtos.StringField.newBuilder().setValue(decompressAbstract(d.get("value").getAsString())).build());

					else

						metadata.addDescription(FieldTypeProtos.StringField.newBuilder().setValue(d.get("value").getAsString()).build());

				}

		);

		//Adding Journal

		final String publisher = getStringValue(rootElement, "publisher");

		if (StringUtils.isNotBlank(publisher)) {

			final ResultProtos.Result.Journal.Builder journal = ResultProtos.Result.Journal.newBuilder().setName(publisher);

			if (hasJSONArrayField(rootElement, "issn")) {

				StreamUtils.toStream(rootElement.getAsJsonArray("issn").iterator())

						.map(JsonElement::getAsJsonObject)

						.forEach(it -> {

							final String issntype = getStringValue(it, "type");

							final String value = getStringValue(it, "value");

							if ("electronic".equals(issntype)) {

								journal.setIssnOnline(value);

							}

							if ("print".equals(issntype))

								journal.setIssnPrinted(value);

						});

			}

			metadata.setJournal(journal.build());

		}

		metadata.setResulttype(getQualifier(getDefaultResulttype(cobjValue), "dnet:result_typologies"));

		result.setMetadata(metadata.build());

		entity.setResult(result.build());

		oaf.setEntity(entity.build());

		//System.out.println(JsonFormat.printToString(oaf.build()));

		final List<AtomicAction> actionList = new ArrayList<>();

		if (!onlyOrganization)

			actionList.add(factory.createAtomicAction(setName, agent, oaf.getEntity().getId(), "result", "body", oaf.build().toByteArray()));

		if (!authorsOrganizations.getValue().isEmpty()) {

			authorsOrganizations.getValue().forEach(o ->

			{

				actionList.add(factory.createAtomicAction(setName, agent, o.getEntity().getId(), "organization", "body", o.toByteArray()));

				if (!onlyOrganization)

					actionList.addAll(createPublicationOrganizationRelation(oaf.build(), o, factory, setName, agent));

				final String gridOrganization = getSimilarGridOrganization(o.getEntity());

				if (gridOrganization != null) {

					actionList.add(factory

							.createAtomicAction(setName, agent, o.getEntity().getId(), "organizationOrganization_dedupSimilarity_isSimilarTo", gridOrganization,

									"".getBytes()));

					actionList.add(factory

							.createAtomicAction(setName, agent, gridOrganization, "organizationOrganization_dedupSimilarity_isSimilarTo", o.getEntity().getId(),

									"".getBytes()));

				}

			});

		}

		return actionList;

	}

	private static String getSimilarGridOrganization(final OafProtos.OafEntity organization) {

		final List<FieldTypeProtos.StructuredProperty> pidList = organization.getPidList();

		if (pidList != null) {

			for (FieldTypeProtos.StructuredProperty p : pidList) {

				if (p.getQualifier().getClassname().equals("grid")) {

					return "20|grid________" + SEPARATOR + AbstractDNetXsltFunctions.md5(p.getValue());

				}

			}

		}

		return null;

	}

	private static List<AtomicAction> createPublicationOrganizationRelation(final OafProtos.Oaf publication,

			final OafProtos.Oaf organization,

			final ActionFactory factory,

			final String setName,

			final Agent agent) {

		List<AtomicAction> result = new ArrayList<>();

		final OafProtos.Oaf.Builder roaf = OafProtos.Oaf.newBuilder();

		roaf.setKind(KindProtos.Kind.relation);

		roaf.setDataInfo(FieldTypeProtos.DataInfo.newBuilder()

				.setInvisible(false)

				.setDeletedbyinference(false)

				.setInferred(false)

				.setTrust("0.9")

				.setProvenanceaction(getQualifier("sysimport:actionset", "dnet:provenanceActions"))

				.build());

		final OafProtos.OafRel.Builder rel = OafProtos.OafRel.newBuilder();

		rel.setRelType(RelTypeProtos.RelType.resultOrganization);

		rel.setSubRelType(RelTypeProtos.SubRelType.affiliation);

		//Create a relation Result --> Organization

		rel.setSource(publication.getEntity().getId());

		rel.setTarget(organization.getEntity().getId());

		rel.setRelClass(ResultOrganization.Affiliation.RelName.hasAuthorInstitution.toString());

		final ResultOrganization.Builder rel_instance = ResultOrganization.newBuilder();

		final ResultOrganization.Affiliation.Builder affiliationRel = ResultOrganization.Affiliation.newBuilder();

		affiliationRel.setRelMetadata(RelMetadataProtos.RelMetadata.newBuilder()

				.setSemantics(getQualifier("hasAuthorInstitution", "dnet:result_organization_relations"))

				.build());

		rel_instance.setAffiliation(affiliationRel.build());

		rel.setResultOrganization(rel_instance.build());

		rel.addCollectedfrom(FieldTypeProtos.KeyValue.newBuilder()

				.setValue(datasources.get(MAG.toLowerCase()).getKey())

				.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + AbstractDNetXsltFunctions

						.md5(StringUtils.substringAfter(datasources.get(MAG.toLowerCase()).getValue(), SEPARATOR)))

				.build());

		rel.setChild(false);

		roaf.setRel(rel.build());

		result.add(factory.createAtomicAction(setName, agent, publication.getEntity().getId(), "resultOrganization_affiliation_hasAuthorInstitution",

				organization.getEntity().getId(), roaf.build().toByteArray()));

		//Create a relation Organization --> Result

		rel.setTarget(publication.getEntity().getId());

		rel.setSource(organization.getEntity().getId());

		rel.setRelClass(ResultOrganization.Affiliation.RelName.isAuthorInstitutionOf.toString());

		affiliationRel.setRelMetadata(RelMetadataProtos.RelMetadata.newBuilder()

				.setSemantics(getQualifier("isAuthorInstitutionOf", "dnet:result_organization_relations"))

				.build());

		rel_instance.setAffiliation(affiliationRel.build());

		rel.setResultOrganization(rel_instance.build());

		roaf.setRel(rel.build());

		result.add(factory.createAtomicAction(setName, agent, organization.getEntity().getId(), "resultOrganization_affiliation_isAuthorInstitutionOf",

				publication.getEntity().getId(), roaf.build().toByteArray()));

		return result;

	}

	private static boolean hasJSONArrayField(final JsonObject root, final String key) {

		return root.has(key) && root.get(key).isJsonArray();

	}

	private static void settingRelevantDate(JsonObject rootElement,

			ResultProtos.Result.Metadata.Builder metadata,

			final String jsonKey,

			final String dictionaryKey,

			final boolean addToDateOfAcceptance) {

		//Adding date

		String date = getStringValue(rootElement, jsonKey);

		if (date == null)

			return;

		if (date.length() == 4) {

			date += "-01-01";

		}

		if (isValidDate(date)) {

			if (addToDateOfAcceptance)

				metadata.setDateofacceptance(FieldTypeProtos.StringField.newBuilder().setValue(date).build());

			metadata.addRelevantdate(FieldTypeProtos.StructuredProperty.newBuilder()

					.setValue(date)

					.setQualifier(getQualifier(dictionaryKey, "dnet:dataCite_date"))

					.build());

		}

	}

	public static FieldTypeProtos.KeyValue extractIdentifier(final String value) {

		FieldTypeProtos.KeyValue.Builder pid = FieldTypeProtos.KeyValue.newBuilder();

		if (StringUtils.contains(value, "orcid.org")) {

			return pid.setValue(value.replaceAll("https://orcid.org/", ""))

					.setKey(ORCID).build();

		}

		if (StringUtils.contains(value, "academic.microsoft.com/#/detail")) {

			return pid.setValue(value.replaceAll("https://academic.microsoft.com/#/detail/", ""))

					.setKey("MAG Identifier").build();

		}

		return pid.setValue(value)

				.setKey("URL").build();

	}

	public static OafProtos.Oaf createOrganizationFromJSON(final JsonObject affiliation) {

		final Map<String, FieldTypeProtos.Qualifier> affiliationIdentifiers = new HashMap<>();

		final List<String> magId = new ArrayList<>();

		getArrayObjects(affiliation, "identifiers").forEach(it -> {

			if (StringUtils.contains(it.get("value").getAsString(), "academic.microsoft.com")) {

				affiliationIdentifiers.put(it.get("value").getAsString(), affiliationPIDType.get(MAG));

				magId.add(it.get("value").getAsString());

			} else

				affiliationIdentifiers.put(it.get("value").getAsString(), affiliationPIDType.get(it.get("schema").getAsString()));

		});

		if (magId.size() > 0) {

			final String microsoftID = magId.get(0);

			OafProtos.Oaf.Builder oaf = OafProtos.Oaf.newBuilder();

			oaf.setKind(KindProtos.Kind.entity);

			OafProtos.OafEntity.Builder entity = OafProtos.OafEntity.newBuilder();

			entity.setType(TypeProtos.Type.organization);

			entity.setId("20|microsoft___" + SEPARATOR + AbstractDNetXsltFunctions.md5(microsoftID));

			final String id = datasources.get(affiliation.get("provenance").getAsString().toLowerCase()).getValue();

			final String name = datasources.get(affiliation.get("provenance").getAsString().toLowerCase()).getKey();

			if (StringUtils.isNotBlank(id) && StringUtils.isNotBlank(name)) {

				final FieldTypeProtos.KeyValue collectedFrom = FieldTypeProtos.KeyValue.newBuilder()

						.setValue(name)

						.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + AbstractDNetXsltFunctions.md5(StringUtils.substringAfter(id, SEPARATOR)))

						.build();

				entity.addCollectedfrom(collectedFrom);

			} else {

				return null;

			}

			entity.addOriginalId(microsoftID);

			affiliationIdentifiers.forEach((key, value) -> entity.addPid(

					FieldTypeProtos.StructuredProperty.newBuilder()

							.setQualifier(value)

							.setValue(key)

							.build()));

			final OrganizationProtos.Organization.Builder organization = OrganizationProtos.Organization.newBuilder();

			organization.setMetadata(OrganizationProtos.Organization.Metadata.newBuilder()

					.setWebsiteurl(FieldTypeProtos.StringField.newBuilder().setValue(affiliation.get("official-page").getAsString()).build())

					.setLegalname(FieldTypeProtos.StringField.newBuilder().setValue(affiliation.get("value").getAsString()).build())

					.build());

			entity.setOrganization(organization);

			oaf.setEntity(entity);

			oaf.setDataInfo(FieldTypeProtos.DataInfo.newBuilder()

					.setInvisible(false)

					.setDeletedbyinference(false)

					.setInferred(false)

					.setTrust("0.9")

					.setProvenanceaction(getQualifier("sysimport:actionset", "dnet:provenanceActions"))

					.build());

			return oaf.build();

		}

		return null;

	}

	public static Pair<List<FieldTypeProtos.Author>, Collection<OafProtos.Oaf>> createAuthorsOrganization(final JsonObject root) {

		final Map<String, OafProtos.Oaf> affiliations = new HashMap<>();

		List<JsonObject> authors = getArrayObjects(root, "authors");

		final AtomicInteger counter = new AtomicInteger(1);

		List<FieldTypeProtos.Author> collect = authors.stream().map(author -> {

			final String given = getStringValue(author, "given");

			final String family = getStringValue(author, "family");

			String fullname = getStringValue(author, "fullname");

			if (StringUtils.isBlank(fullname) && StringUtils.isNotBlank(given) && StringUtils.isNotBlank(family)) {

				fullname = String.format("%s %s", given, family);

			}

			if (!isValidAuthorName(fullname, null)) {

				return null;

			}

			final FieldTypeProtos.Author.Builder abuilder = FieldTypeProtos.Author.newBuilder();

			if (StringUtils.isNotBlank(given))

				abuilder.setName(given);

			if (StringUtils.isNotBlank(family))

				abuilder.setSurname(family);

			if (StringUtils.isNotBlank(fullname))

				abuilder.setFullname(fullname);

			final List<JsonObject> identifiers = getArrayObjects(author, "identifiers");

			final List<JsonObject> authorAffiliation = getArrayObjects(author, "affiliations");

			authorAffiliation.forEach(it ->

			{

				OafProtos.Oaf org = createOrganizationFromJSON(it);

				if (org != null) {

					affiliations.put(org.getEntity().getId(), org);

					abuilder.addAffiliation(org.getEntity().getOrganization().getMetadata().getLegalname());

				}

			});

			identifiers.stream().map(id -> {

				final String value = id.get("value").getAsString();

				return extractIdentifier(value);

			}).collect(

					Collectors.toMap(

							FieldTypeProtos.KeyValue::getKey,

							Function.identity(),

							(a, b) -> a

					)).values().forEach(abuilder::addPid);

			abuilder.setRank(counter.getAndIncrement());

			return abuilder.build();

		}).filter(Objects::nonNull).collect(Collectors.toList());

		return new Pair<>(collect, affiliations.values());

	}