D-Net

package eu.dnetlib.data.utils;

import java.net.URI;

import java.net.URISyntaxException;

import java.security.KeyManagementException;

import java.security.KeyStoreException;

import java.security.NoSuchAlgorithmException;

import java.util.concurrent.TimeUnit;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.http.conn.ssl.SSLConnectionSocketFactory;

import org.apache.http.impl.client.HttpClientBuilder;

import org.apache.http.ssl.SSLContextBuilder;

import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;

import org.springframework.web.client.RestTemplate;

import eu.dnetlib.data.mdstore.plugins.GenericDoiMdstorePlugin;

public class HttpFetcher {

	private static final Log log = LogFactory.getLog(GenericDoiMdstorePlugin.class);

	public static final int MAX_NUMBER_OF_ATTEMPTS = 10;

	private static final int INTERVAL_MILLIS = 20000;

	private static final SSLContextBuilder sslContextBuilder = new SSLContextBuilder();

	private static SSLConnectionSocketFactory sslSocketFactory;

	static {

		try {

			sslContextBuilder.loadTrustMaterial(null, (chain, authType) -> true);

			sslSocketFactory = new SSLConnectionSocketFactory(sslContextBuilder.build());

		} catch (final NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) {

			log.error(e);;

		}

	}

	private static final HttpComponentsClientHttpRequestFactory httpRequestFactory = new HttpComponentsClientHttpRequestFactory(HttpClientBuilder

			.create()

			.setConnectionTimeToLive(0, TimeUnit.MILLISECONDS)

			.setMaxConnPerRoute(1)

			.setMaxConnTotal(1)

			.disableAutomaticRetries()

			.disableConnectionState()

			.setSSLSocketFactory(sslSocketFactory)

			.build());

	public static String fetch(final String url) throws URISyntaxException {

		return fetch(new URI(url), MAX_NUMBER_OF_ATTEMPTS);

	}

	public static String fetch(final URI url) {

		return fetch(url, MAX_NUMBER_OF_ATTEMPTS);

	}

	public static String fetch(final URI url, final int attempts) {

		if (attempts == 0) { throw new RuntimeException("Max number of attempts reached, downloading url: " + url); }

		try {

			return (new RestTemplate(httpRequestFactory)).getForObject(url, String.class);

		} catch (final Exception e) {

			try {

				log.error("Error downloading url: " + url + " - " + e.getMessage());

				Thread.sleep(INTERVAL_MILLIS);

				return fetch(url, attempts - 1);

			} catch (final InterruptedException e1) {

				throw new RuntimeException(e1);

			}

		}

	}

}

import java.util.Map;

	protected void reconfigure(final Map<String, String> params) {}

	protected void reconfigure(final Map<String, String> params) {

package eu.dnetlib.data.mdstore.plugins;

import java.io.StringReader;

import java.net.URISyntaxException;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.Objects;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

import java.util.stream.Collectors;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Autowired;

import eu.dnetlib.data.mdstore.plugins.objects.CnrCollection;

import eu.dnetlib.data.mdstore.plugins.objects.MdRecord;

import eu.dnetlib.data.utils.HttpFetcher;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.rmi.enabling.ISLookUpService;

public class EnrichCollectionPlugin extends MdRecordPlugin {

	private Map<String, CnrCollection> colls = new HashMap<>();

	private static final Log log = LogFactory.getLog(EnrichCollectionPlugin.class);

	@Autowired

	private UniqueServiceLocator serviceLocator;

	@Override

	protected void reconfigure(final Map<String, String> params) {

		params.entrySet().forEach(e -> log.info(String.format("******************* %s -> %s", e.getKey(), e.getValue())));

		try {

			final String profile = serviceLocator.getService(ISLookUpService.class).getResourceProfile(params.get("dsId"));

			final Document doc = new SAXReader().read(new StringReader(profile));

			final String baseUrl = doc.valueOf("//INTERFACE[@id='" + params.get("dsInterface") + "']/BASE_URL").trim();

			colls.clear();

			colls.putAll(listOaiCollections(baseUrl + "?verb=ListSets").stream()

					.map(this::createCollection)

					.filter(Objects::nonNull)

					.collect(Collectors.toMap(CnrCollection::getCode, o -> o)));

		} catch (final Exception e) {

			log.error("Error evaluating ListSets", e);

			throw new RuntimeException("Error evaluating ListSets", e);

		}

	}

	@Override

	protected boolean updateRecord(final MdRecord record) {

		for (final CnrCollection c : record.getInCollections()) {

			if (colls.containsKey(c.getCode())) {

				c.setName(colls.get(c.getCode()).getName());

				c.setAcronym(colls.get(c.getCode()).getAcronym());

			}

		}

		return true;

	}

	@SuppressWarnings("unchecked")

	private List<Node> listOaiCollections(final String listSetsUrl) {

		try {

			final SAXReader reader = new SAXReader();

			final String s = HttpFetcher.fetch(listSetsUrl);

			final Document doc = reader.read(new StringReader(s));

			return doc.selectNodes("//*[local-name() = 'ListSets']/*[local-name() = 'set']");

		} catch (final DocumentException | URISyntaxException e) {

			log.error("Error listing sets from url: " + listSetsUrl, e);

			return new ArrayList<>();

		}

	}

	private CnrCollection createCollection(final Node n) {

		final CnrCollection c = new CnrCollection();

		final String code = n.valueOf("./*[local-name() = 'setSpec']").trim();

		c.setCode(n.valueOf("./*[local-name() = 'setSpec']").trim());

		if (code.equalsIgnoreCase("openaire")) {

			c.setAcronym("openaire");

			c.setName("Openaire Collection");

		} else {

			final String desc = n.valueOf("./*[local-name() = 'setDescription']").trim();

			final Matcher m = Pattern.compile("^Prodotti della ricerca di (.+) \\- (.+)$").matcher(desc);

			if (m.matches()) {

				c.setAcronym(m.group(1));

				c.setName(m.group(2));

			} else {

				c.setAcronym(desc);

				c.setName(desc);

			}

		}

		return c;

	}

}

import eu.dnetlib.data.utils.HttpFetcher;

			return HttpFetcher.fetch(prepareURI(doi));

	protected void reconfigure(final Map<String, String> params) {

	@XmlElementWrapper(name = "collections")

	@XmlElement(name = "inCollection")

	private Set<CnrCollection> inCollections = new LinkedHashSet<>();

	public void setCnrPersons(final Set<CnrPerson> cnrPersons) {

		this.cnrPersons = cnrPersons;

	}

	public Set<CnrCollection> getInCollections() {

		return inCollections;

	}

	public void setInCollections(final Set<CnrCollection> inCollections) {

		this.inCollections = inCollections;

	}

import eu.dnetlib.data.utils.HttpFetcher;

			final String s = HttpFetcher.fetch(url);

			final Document doc = reader.read(new StringReader(s));

package eu.dnetlib.data.mdstore.plugins.objects;

import javax.xml.bind.annotation.XmlAccessType;

import javax.xml.bind.annotation.XmlAccessorType;

import javax.xml.bind.annotation.XmlElement;

@XmlAccessorType(XmlAccessType.FIELD)

public class CnrCollection {

	@XmlElement(name = "code")

	public String code;

	@XmlElement(name = "name")

	public String name;

	@XmlElement(name = "acronym")

	public String acronym;

	public String getCode() {

		return code;

	}

	public void setCode(final String code) {

		this.code = code;

	}

	public String getAcronym() {

		return acronym;

	}

	public void setAcronym(final String acronym) {

		this.acronym = acronym;

	}

	public String getName() {

		return name;

	}

	public void setName(final String name) {

		this.name = name;

	}

}

import java.util.Map;

	protected void reconfigure(final Map<String, String> params) {}

		reconfigure(params);

	abstract protected void reconfigure(Map<String, String> params);

	<bean id="enrichCollectionsMDstorePlugin" class="eu.dnetlib.data.mdstore.plugins.EnrichCollectionPlugin" />

                        <ARC to="enrichCollections"/>

                    </ARCS>

                </NODE>

                <NODE name="enrichCollections" type="LaunchWorkflowTemplate">

                    <DESCRIPTION>Enrich with collections using OAI sets</DESCRIPTION>

                    <PARAMETERS>

                        <PARAM name="wfTemplateId" value="f0ffb1b2-1518-4087-aa6d-046f507a50ca_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ==" />

                        <PARAM name="wfTemplateParams">

                            <MAP>

                                <ENTRY key="mdId"         ref="cleanMdstoreId" />

                                <ENTRY key="dsId"         value="$dsId$" />

                                <ENTRY key="dsInterface"  value="$interface$" />

                            </MAP>

                        </PARAM>

                    </PARAMETERS>

                    <ARCS>

                    <collections>

                    		<xsl:for-each select="//oai:setSpec">

	                        <inCollection>

	                        		<code><xsl:value-of select="normalize-space(.)" /></code>

	                        		<name />

	                             <acronym />

	                        </inCollection>

	                    </xsl:for-each>

                    </collections>

<RESOURCE_PROFILE>

	<HEADER>

		<RESOURCE_IDENTIFIER value="f0ffb1b2-1518-4087-aa6d-046f507a50ca_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>

		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>

		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>

		<RESOURCE_URI value=""/>

		<DATE_OF_CREATION value="2001-12-31T12:00:00"/>

	</HEADER>

	<BODY>

		<CONFIGURATION>

			<PARAMETERS>

				<PARAM name="mdId" description="MDStoreId" required="true" type="string"/>

				<PARAM name="dsId" description="DatasourceId" required="true" type="string"/>

				<PARAM name="dsInterface" description="DatasourceInterface" required="true" type="string"/>

			</PARAMETERS>

			<WORKFLOW>

				<NODE name="enrich_colls" type="RunMDStorePlugin" isStart="true">

					<DESCRIPTION>Enrich with collections (OAI sets)</DESCRIPTION>

					<PARAMETERS>

						<PARAM name="pluginName" value="enrichCollectionsMDstorePlugin"/>

						<PARAM name="parameters">

							<MAP>

								<ENTRY key="mdId" ref="mdId" />

								<ENTRY key="dsId" ref="dsId" />

								<ENTRY key="dsInterface" ref="dsInterface" />

							</MAP>

						</PARAM>

					</PARAMETERS>

					<ARCS>

						<ARC to="success"/>

					</ARCS>

				</NODE>	

			</WORKFLOW>

		</CONFIGURATION>

	</BODY>

</RESOURCE_PROFILE>