D-Net

package eu.dnetlib.data.mdstore.plugins;

import java.io.StringReader;

import java.net.URI;

import java.net.URISyntaxException;

import java.security.KeyManagementException;

import java.security.KeyStoreException;

import java.security.NoSuchAlgorithmException;

import java.util.Map;

import java.util.concurrent.TimeUnit;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.http.conn.ssl.SSLConnectionSocketFactory;

import org.apache.http.impl.client.HttpClientBuilder;

import org.apache.http.ssl.SSLContextBuilder;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;

import org.springframework.web.client.RestTemplate;

import com.mongodb.BasicDBObject;

import com.mongodb.DBObject;

import com.mongodb.client.MongoCollection;

import eu.dnetlib.data.mdstore.modular.mongodb.MongoMDStore;

import eu.dnetlib.rmi.data.MDStoreServiceException;

public abstract class GenericDoiMdstorePlugin extends AbstractIstiMDStorePlugin {

	private static final Log log = LogFactory.getLog(GenericDoiMdstorePlugin.class);

	private static final int MAX_NUMBER_OF_ATTEMPTS = 10;

	private static final int INTERVAL_MILLIS = 20000;

	private static final SSLContextBuilder sslContextBuilder = new SSLContextBuilder();

	private static SSLConnectionSocketFactory sslSocketFactory;

	static {

		try {

			sslContextBuilder.loadTrustMaterial(null, (chain, authType) -> true);

			sslSocketFactory = new SSLConnectionSocketFactory(sslContextBuilder.build());

		} catch (final NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) {

			log.error(e);;

		}

	}

	private static final HttpComponentsClientHttpRequestFactory httpRequestFactory = new HttpComponentsClientHttpRequestFactory(HttpClientBuilder

			.create()

			.setConnectionTimeToLive(0, TimeUnit.MILLISECONDS)

			.setMaxConnPerRoute(1)

			.setMaxConnTotal(1)

			.disableAutomaticRetries()

			.disableConnectionState()

			.setSSLSocketFactory(sslSocketFactory)

			.build());

	@Override

	public void process(final MongoMDStore store, final Map<String, String> params) throws MDStoreServiceException {

		final MongoCollection<DBObject> collPubs = store.getCollection();

		int count = 0;

		for (final DBObject obj : collPubs.find()) {

			try {

				final String recordId = obj.get("id").toString();

				final Document doc = (new SAXReader()).read(new StringReader(obj.get("body").toString()));

				for (final Object o : doc.selectNodes("//*[local-name()='alternateIdentifier' and @alternateIdentifierType='doi']")) {

					final String doi = ((Node) o).getText().trim();

					log.debug("  Record " + recordId + " has doi " + doi);

					final String response = download(doi);

					if ((response != null) && updateDocument(doc, response)) {

						collPubs.updateOne(new BasicDBObject("id", recordId), new BasicDBObject("$set", new BasicDBObject("body", doc.asXML())));

						count++;

					}

				}

			} catch (final DocumentException e) {

				log.warn("Problem parsing a mdstore record");

			}

		}

		log.info("Number of patched records: " + count);

		touch(store);

	}

	abstract protected boolean updateDocument(Document doc, String response);

	abstract protected URI prepareURI(String doi) throws URISyntaxException;

	private String download(final String doi) {

		try {

			return fetchUrl(prepareURI(doi), MAX_NUMBER_OF_ATTEMPTS);

		} catch (final URISyntaxException e) {

			log.error("Error resolving doi: " + doi, e);

			return null;

		}

	}

	private static String fetchUrl(final URI url, final int attempts) {

		if (attempts == 0) { throw new RuntimeException("Max number of attempts reached, downloading url: " + url); }

		try {

			return (new RestTemplate(httpRequestFactory)).getForObject(url, String.class);

		} catch (final Exception e) {

			try {

				log.error("Error downloading url: " + url + " - " + e.getMessage());

				Thread.sleep(INTERVAL_MILLIS);

				return fetchUrl(url, attempts - 1);

			} catch (final InterruptedException e1) {

				throw new RuntimeException(e1);

			}

		}

	}

}