D-Net

package eu.dnetlib.data.mdstore.plugins;

import java.io.StringReader;

import java.net.URI;

import java.net.URISyntaxException;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.Set;

import java.util.stream.Collectors;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.Element;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Value;

import com.google.common.base.Splitter;

import com.mongodb.MongoClient;

import com.mongodb.client.MongoCollection;

import eu.dnetlib.data.mdstore.plugins.objects.MdRecord;

import eu.dnetlib.data.mdstore.plugins.objects.MyURL;

import eu.dnetlib.data.mdstore.plugins.objects.Project;

import eu.dnetlib.data.utils.XsltFunctions;

public class EnrichOpenairePlugin extends GenericDoiMdstorePlugin {

	private static final Log log = LogFactory.getLog(EnrichOpenairePlugin.class);

	@Value("${plugin.enrich.publications.openaire.url}")

	private String baseUrl;

	@Value("${plugin.enrich.openaire.datasources.blacklist}")

	private String datasourceBlackList;

	@Autowired

	private MongoClient mongoClient;

	private Map<String, Counter> counters = new HashMap<>();

	@Override

	protected URI prepareURI(final String doi) throws URISyntaxException {

		return new URI(String.format(baseUrl, doi));

	}

	@Override

	protected MongoCollection<org.bson.Document> getCacheCollection() {

		return mongoClient.getDatabase("API_CACHES").getCollection("OPENAIRE_API_CACHE");

	}

	@Override

	protected void reconfigure(final Map<String, String> params) {

		counters.clear();

		counters.put("subjects", new Counter());

		counters.put("citations", new Counter());

		counters.put("urls", new Counter());

		counters.put("projects", new Counter());

		counters.put("dois", new Counter());

	}

	@Override

	protected void resetConfiguration() {

		log.info("***** Openaire Enrichment - subjects  : " + counters.get("subjects"));

		log.info("***** Openaire Enrichment - citations : " + counters.get("citations"));

		log.info("***** Openaire Enrichment - urls      : " + counters.get("urls"));

		log.info("***** Openaire Enrichment - projects  : " + counters.get("projects"));

		log.info("***** Openaire Enrichment - dois      : " + counters.get("dois"));

		counters.clear();

	}

	@Override

	protected boolean updateDocument(final MdRecord doc, final String response) {

		counters.get("subjects").incrementBefore(doc.getSubjects().size());

		counters.get("citations").incrementBefore(doc.getCitations().size());

		counters.get("urls").incrementBefore(doc.getUrls().size());

		counters.get("projects").incrementBefore(doc.getProjects().size());

		counters.get("dois").incrementBefore(doc.getDois().size());

		try {

			final Document docRes = (new SAXReader()).read(new StringReader(response));

			final List<?> results = docRes.selectNodes("/response/results/result");

			if (results.size() == 1) {

				final Node n = (Node) results.get(0);

				updateSubjects(doc, n);

				updateCitations(doc, n);

				updateUrls(doc, n);

				updateProjects(doc, n);

				updateDois(doc, n);

				updateBestRights(doc);

				return true;

			} else if (results.size() == 1) {

				log.warn("Too many responses");

			}

		} catch (final DocumentException e) {

			log.warn("Invalid response", e);

		} finally {

			counters.get("subjects").incrementAfter(doc.getSubjects().size());

			counters.get("citations").incrementAfter(doc.getCitations().size());

			counters.get("urls").incrementAfter(doc.getUrls().size());

			counters.get("projects").incrementAfter(doc.getProjects().size());

			counters.get("dois").incrementAfter(doc.getDois().size());

		}

		return false;

	}

	private void updateSubjects(final MdRecord doc, final Node n) {

		final Set<String> subjects = doc.getSubjects()

				.stream()

				.map(EnrichOpenairePlugin::cleanSubject)

				.flatMap(List::stream)

				.collect(Collectors.toSet());

		for (final Object o : n.selectNodes(".//subject[@classid='keyword']")) {

			subjects.addAll(cleanSubject(((Node) o).getText().trim()));

		}

		doc.setSubjects(subjects);

	}

	public static List<String> cleanSubject(final String s) {

		if (s.isEmpty()) {

			return new ArrayList<>();

		} else if (s.startsWith("info:eu-repo/classification/msc/")) {

			return new ArrayList<>();

		} else if (s.startsWith("info:eu-repo/classification/acm/")) {

			return Arrays.asList(s.replaceFirst("info:eu-repo/classification/acm/", ""));

		} else if (s.contains(";")) {

			return Splitter.on(";").trimResults().omitEmptyStrings().splitToList(s);

		} else if (s.contains(",")) {

			return Splitter.on(",").trimResults().omitEmptyStrings().splitToList(s);

		} else {

			return Arrays.asList(s);

		}

	}

	private void updateCitations(final MdRecord doc, final Node n) {

		doc.getCitations().clear();

		for (final Object o : n.selectNodes(".//citations/citation/rawText")) {

			doc.getCitations().add(((Node) o).getText());

		}

		for (final Object o : n.selectNodes(".//references/reference/rawText")) {

			doc.getCitations().add(((Node) o).getText());

		}

	}

	private void updateUrls(final MdRecord doc, final Node n) {

		doc.getUrls().addAll(doc.getUrls());

		final Set<String> blacklist =

				Arrays.stream(datasourceBlackList.split(",")).map(String::trim).filter(StringUtils::isNotBlank).collect(Collectors.toSet());

		for (final Object oin : n.selectNodes(".//instance")) {

			final String hostedByid = ((Element) oin).valueOf("./hostedby/@id").trim();

			if (!blacklist.contains(hostedByid)) {

				final String hostedBy = ((Element) oin).valueOf("./hostedby/@name").trim();

				final String rights = ((Element) oin).valueOf("./accessright/@classname").trim();

				for (final Object ourl : ((Element) oin).selectNodes("./webresource/url")) {

					final String url = ((Node) ourl).getText().trim();

					final String name =

							hostedBy.equalsIgnoreCase("Unknown") || hostedBy.equalsIgnoreCase("Unknown Repository") ? XsltFunctions.serverName(url) : hostedBy;

					final MyURL u = new MyURL(url, name, rights);

					doc.getUrls().remove(u);

					doc.getUrls().add(u);

				}

			}

		}

	}

	private void updateProjects(final MdRecord doc, final Node n) {

		for (final Object op : n.selectNodes(".//rels/rel[./to/@type='project']")) {

			final Node p = (Node) op;

			final String name = p.valueOf("./title").trim();

			if (StringUtils.isNotBlank(name) && !name.equalsIgnoreCase("null") && !name.equalsIgnoreCase("unidentified") && !name.equalsIgnoreCase("unknown")) {

				final Project np = new Project();

				np.setOpenaireId(p.valueOf("./to"));

				np.setCode(p.valueOf("./code"));

				np.setName(name);

				np.setAcronym(p.valueOf("./acronym"));

				np.setFunder(p.valueOf(".//funder/@shortname"));

				np.setProgram(p.valueOf(".//funding_level_0/@name"));

				np.setJurisdiction(p.valueOf(".//funder/@jurisdiction"));

				np.setInfoId(XsltFunctions.projectLongId(np.getFunder(), np.getProgram(), np.getCode(), np.getJurisdiction(),

						np.getName(), np.getAcronym()));

				doc.getProjects().add(np);

			}

		}

	}

	private void updateDois(final MdRecord doc, final Node n) {

		for (final Object od : n.selectNodes(".//*[local-name()='result']/pid[@classid='doi']")) {

			final String doi = XsltFunctions.cleanDoi(((Node) od).getText());

			if (StringUtils.isNotBlank(doi)) {

				doc.getDois().add(doi);

			}

		}

	}

	private void updateBestRights(final MdRecord doc) {

		final Set<String> availables = doc.getUrls().stream().map(MyURL::getRights).map(String::toUpperCase).collect(Collectors.toSet());

		if (availables.contains("OPEN ACCESS")) {

			doc.setBestRights("Open Access");

		} else if (availables.contains("EMBARGO")) {

			doc.setBestRights("Embargo");

		} else if (availables.contains("RESTRICTED")) {

			doc.setBestRights("Restricted");

		} else if (availables.contains("CLOSED ACCESS")) {

			doc.setBestRights("Closed Access");

		} else {

			doc.setBestRights("Unknown");

		}

	}

}