D-Net

package eu.dnetlib.data.mdstore.plugins;

import java.io.StringReader;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.Optional;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

import java.util.stream.Collectors;

import org.apache.commons.lang3.math.NumberUtils;

import org.apache.commons.lang3.text.WordUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.cxf.jaxws.JaxWsProxyFactoryBean;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.Element;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Value;

import com.google.common.base.Splitter;

import com.google.common.collect.Lists;

import com.mongodb.BasicDBObject;

import com.mongodb.DBObject;

import com.mongodb.client.MongoCollection;

import eu.dnetlib.clients.pimpa.DataRange;

import eu.dnetlib.clients.pimpa.Laboratory;

import eu.dnetlib.clients.pimpa.Person;

import eu.dnetlib.clients.pimpa.PimpaService;

import eu.dnetlib.clients.pimpa.Year;

import eu.dnetlib.data.mdstore.modular.mongodb.MongoMDStore;

import eu.dnetlib.data.mdstore.plugins.objects.CnrAuthor;

import eu.dnetlib.rmi.data.MDStoreServiceException;

public class EnrichLabsPlugin extends AbstractIstiMDStorePlugin {

	private static final Log log = LogFactory.getLog(EnrichLabsPlugin.class);

	@Value("${plugin.enrich.labs.pimpa.url}")

	private String serviceUrl;

	@SuppressWarnings("unchecked")

	@Override

	public void process(final MongoMDStore store, final Map<String, String> params) throws MDStoreServiceException {

		log.warn("********************************************");

		log.warn("* ENRICH LABS");

		log.warn("********************************************");

		final DataRange res = findDataRange(1980, 2017);

		// CODE -> year -> Lab

		final Map<String, Map<Integer, List<Laboratory>>> labs = new HashMap<>();

		// CODE -> names

		final Map<String, String> names = new HashMap<>();

		for (final Year y : res.getYears()) {

			final Integer year = y.getYear();

			for (final Person p : y.getPersons().getList()) {

				names.putIfAbsent(p.getCode(), WordUtils.capitalize((p.getName() + " " + p.getSurname()).toLowerCase()));

				labs.putIfAbsent(p.getCode(), new HashMap<>());

				labs.get(p.getCode()).putIfAbsent(year, p.getLabs().getList());

			}

		}

		final MongoCollection<DBObject> pubsCollection = store.getCollection();

		for (final DBObject obj : pubsCollection.find()) {

			final String id = obj.get("id").toString();

			log.debug("**********************************************");

			log.debug("Record " + id);

			try {

				final Document doc = (new SAXReader()).read(new StringReader(obj.get("body").toString()));

				resetAffiliations(doc);

				final int publicationYear = NumberUtils.toInt(doc.valueOf("//*[local-name()='date' and @dateType='Accepted']"), 0);

				final List<Element> creatorNodes = doc.selectNodes("//*[local-name() = 'creator']");

				for (final CnrAuthor cnrAuthor : listCnrAuthors(doc)) {

					if (log.isDebugEnabled()) {

						log.debug("    Surname : " + cnrAuthor.getSurname());

						log.debug("    Name    : " + cnrAuthor.getName());

						log.debug("    Date    : " + publicationYear);

					}

					final Element node = findNodeCreator(creatorNodes, cnrAuthor.getName(), cnrAuthor.getSurname());

					if (node != null) {

						log.debug("*** " + node.valueOf("./*[local-name() = 'creatorName']") + " -> " + names.get(cnrAuthor.getCode()));

						if (labs.containsKey(cnrAuthor.getCode()) && labs.get(cnrAuthor.getCode()).containsKey(publicationYear)

								&& names.containsKey(cnrAuthor.getCode())) {

							if (log.isDebugEnabled()) {

								log.debug("    Fullname (PIMPA): " + names.get(cnrAuthor.getCode()));

								log.debug("    Laboratories (PIMPA): "

										+ labs.get(cnrAuthor.getCode()).get(publicationYear).stream().map(Laboratory::getCode).collect(Collectors.joining()));

							}

							node.selectSingleNode("./*[local-name() = 'creatorName']").setText(names.get(cnrAuthor.getCode()));

							for (final Laboratory l : labs.get(cnrAuthor.getCode()).get(publicationYear)) {

								final Element affNode = node.addElement("affiliation");

								affNode.setText("ISTI-CNR");

								affNode.addAttribute("group", l.getDescription());

								affNode.addAttribute("type", l.getType());

								affNode.addAttribute("groupAcronym", l.getCode());

								affNode.addAttribute("code", cnrAuthor.getCode());

							}

						} else {

							log.warn("CNR User not found in pimpa: " + cnrAuthor);

							final Element affNode = node.addElement("affiliation");

							node.selectSingleNode("./*[local-name() = 'creatorName']").setText(cnrAuthor.getFullname());

							affNode.addAttribute("code", cnrAuthor.getCode());

						}

					} else {

						log.warn("----");

						log.warn("Match non found for user : " + names.get(cnrAuthor.getCode()));

						log.warn("                  record : " + id);

						log.warn("         list of authors : " + ((List<Element>) doc.selectNodes("//*[local-name() = 'creatorName']"))

								.stream()

								.map(Element::getText)

								.collect(Collectors.joining(", ")));

					}

					log.debug("    ---");

					pubsCollection.updateOne(new BasicDBObject("id", id), new BasicDBObject("$set", new BasicDBObject("body", doc.asXML())));

					// For creators-cnrusers matching

					// https://svn.driver.research-infrastructures.eu/driver/private/claudio.atzori/dnet-dedup-preprocess/trunk

					// https://svn.driver.research-infrastructures.eu/driver/private/claudio.atzori/dnet-dedup-preprocess/trunk/src/main/java/eu/dnetlib/MatchParser.java

				}

			} catch (final DocumentException e) {

				log.warn("Problem parsing a mdstore record");

			}

		}

	}

	private void resetAffiliations(final Document doc) {

		for (final Object n : doc.selectNodes("//*[local-name() = 'creator']/*[local-name() = 'affiliation']")) {

			((Node) n).detach();

		}

	}

	protected List<CnrAuthor> listCnrAuthors(final Document doc) {

		final List<CnrAuthor> res = new ArrayList<>();

		for (final Object o : doc.selectNodes("//*[local-name()='person']/*[local-name()='infoId']")) {

			final String s = ((Element) o).getText();

			final Pattern pattern = Pattern.compile("info:cnr-pdr\\/author\\/(.+):(.+)\\/(.+)\\/(.+)");

			final Matcher matcher = pattern.matcher(s);

			if (matcher.find()) {

				if (matcher.group(1).equals("matricola")) {

					final CnrAuthor auth = new CnrAuthor();

					auth.setCode(matcher.group(2));

					auth.setSurname(matcher.group(3));

					auth.setName(matcher.group(4));

					res.add(auth);

				}

			}

		}

		return res;

	}

	protected Element findNodeCreator(final List<Element> nodes, final String name, final String surname) {

		final Iterable<String> s1 = cleanValue(name + " " + surname);

		final Optional<Element> res = nodes.stream()

				.filter(n -> {

					final Iterable<String> s2 = cleanValue(n.valueOf("./*[local-name() = 'creatorName']"));

					return verifyMatch(s1, s2) || verifyMatch(s2, s1);

				})

				.findFirst();

		if (res.isPresent()) {

			nodes.remove(res.get());

			return res.get();

		}

		return null;

	}

	protected boolean verifyMatch(final Iterable<String> s1, final Iterable<String> s2) {

		final ArrayList<String> cs1 = new ArrayList<>();

		final ArrayList<String> cs2 = Lists.newArrayList(s2);

		final int start = cs2.size();

		for (final String s : s1) {

			if (!cs2.remove(s)) {

				cs1.add(s);

			}

		}

		for (final String s : cs1) {

			cs2.remove(s.substring(0, 1));

		}

		return (start - cs2.size()) >= 2;

	}

	protected Iterable<String> cleanValue(final String s) {

		return Splitter.on(" ").omitEmptyStrings().trimResults().split(s.toLowerCase()

				.replaceAll("[àáâaäææãā]", "a")

				.replaceAll("[èéêëēėę]", "e")

				.replaceAll("[îïíīįì]", "i")

				.replaceAll("[ôöòóœøōõ]", "o")

				.replaceAll("[ûüùúū]", "u")

				.replaceAll("[^a-z\\s]", ""));

	}

	private DataRange findDataRange(final int from, final int to) {

		final JaxWsProxyFactoryBean factory = new JaxWsProxyFactoryBean();

		factory.setServiceClass(PimpaService.class);

		factory.setAddress(serviceUrl);

		final PimpaService pimpaService = (PimpaService) factory.create();

		return pimpaService.getDataRange(from, to);

	}

	/*

	 * private PersonLabs findLabForUser(final String user, final int year) { final JaxWsProxyFactoryBean factory = new

	 * JaxWsProxyFactoryBean(); factory.setServiceClass(PimpaService.class);

	 * factory.setAddress("http://pimpa.isti.cnr.it/PERSONALE/web-services/iop/iop.webservice.php"); final PimpaService pimpaService =

	 * (PimpaService) factory.create(); return pimpaService.getLabs(user, year); }

	 */

}