D-Net

log4j.logger.eu.dnetlib.data.collector.plugins.oai.engine.HttpConnector=FATAL

import java.nio.file.FileSystems;

import java.nio.file.Files;

import java.nio.file.Path;

import java.util.List;

import java.util.stream.Collectors;

import eu.dnetlib.resolver.model.CompletionStatus;

import eu.dnetlib.resolver.parser.ScholixResolverParser;

import org.apache.commons.lang3.tuple.Pair;

import org.apache.http.HttpEntity;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.impl.client.CloseableHttpClient;

import org.apache.http.impl.client.HttpClients;

import org.apache.http.util.EntityUtils;

	public void testDMFParser() throws IOException {

	public void testScholixParser() throws IOException {

		final InputStream resourceAsStream = getClass().getResourceAsStream("/eu/dnetlib/dli/parser/InputRecordScholix.xml");

		String str = IOUtils.toString(resourceAsStream);

		ScholixResolverParser parser = new ScholixResolverParser();

		ResolvedObject object = parser.parseObject(str);

		System.out.println("object = " + object);

		Assert.assertNotNull(object);

		System.out.println("object.getCompletionStatus() = " + object.getCompletionStatus());

		Assert.assertTrue(object.getCompletionStatus().equals(CompletionStatus.incomplete.toString()));

		Assert.assertNotNull(object.getRelations());

		Assert.assertTrue(object.getRelations().size() == 1);

		System.out.println("object.getRelations().get(0) = " + object.getRelations().get(0));

	}

	@Test

	public void testProteinParser() throws IOException {

		final NCBINResolver resolver = new NCBINResolver();

		resolver.setNCBINParser(new NCBINParser());

		final ResolvedObject p02768 = resolver.resolve("U36380", "protein");

		System.out.println(p02768.getDescription());

		System.out.println("p02768 = " + p02768);

	}

	@Test

	public void testNCBIn() throws Exception {

		Path path = FileSystems.getDefault().getPath("/var/lib/dli/elsevier.csv");

		//Files.lines(path).forEach(it -> System.out.println("it = " + it));

		List<Pair<String, String>> elem =

				Files.lines(path).map(it -> it.split("@")).filter(it -> (it != null && it.length == 5)).map(it -> Pair.of(it[3], it[4]))

						.collect(Collectors.toList());

		final NCBINResolver resolver = new NCBINResolver();

		resolver.setNCBINParser(new NCBINParser());

		elem.forEach(it -> {

			if (it.getLeft().equals("ncbi-n")) {

				final ResolvedObject obj = resolver.resolve(it.getRight(), "ncbi-n");

				System.out.println("p02768 = " + obj);

			}

		});

	}

	private String requestUrl(final String url) {

		final CloseableHttpClient httpclient = HttpClients.createDefault();

		try {

			HttpGet httpGet = new HttpGet(url);

			final String out = httpclient.execute(httpGet, response -> {

				int status = response.getStatusLine().getStatusCode();

				if (status >= 200 && status < 300) {

					HttpEntity entity = response.getEntity();

					return entity != null ? EntityUtils.toString(entity) : null;

				} else {

					return null;

				}

			});

			return out;

		} catch (Throwable e) {

			log.error(e);

			return null;

		} finally {

			try {

				httpclient.close();

			} catch (IOException e) {

				log.error("Error on closing httpclient", e);

			}

		}

	}

	@Test

<?xml version="1.0" encoding="UTF-8"?>

<oai:record xmlns:oai="http://www.openarchives.org/OAI/2.0/" xmlns:dri="http://www.driver-repository.eu/namespace/dri"

            xmlns="http://namespace.openaire.eu/">

	<oai:header>

		<dri:objIdentifier>dli::r3d100010134::00002f60593fd1f758fb838fafb46795</dri:objIdentifier>

		<dri:recordIdentifier>oai:pangaea.de:doi:10.1594/PANGAEA.432865</dri:recordIdentifier>

		<dri:dateOfCollection>2016-09-12T16:55:22.27+02:00</dri:dateOfCollection>

		<dri:repositoryId/>

		<dri:datasourceprefix>dli::r3d100010134</dri:datasourceprefix>

		<dri:datasourceID/>

		<identifier xmlns="http://www.openarchives.org/OAI/2.0/"

		>oai:pangaea.de:doi:10.1594/PANGAEA.432865

		</identifier>

		<datestamp xmlns="http://www.openarchives.org/OAI/2.0/">2016-06-25T12:35:00Z</datestamp>

	</oai:header>

	<metadata xmlns="http://www.openarchives.org/OAI/2.0/">

		<resource xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

		          xmlns="http://datacite.org/schema/kernel-3"

		          xsi:schemaLocation="http://datacite.org/schema/kernel-3 http://schema.datacite.org/meta/kernel-3/metadata.xsd">

			<identifier identifierType="DOI">10.1594/PANGAEA.432865</identifier>

			<creators>

				<creator>

					<creatorName>WOCE Sea Level, WSL</creatorName>

					<creatorName>WOCE Sea Level, WSL</creatorName>

				</creator>

			</creators>

			<titles>

				<title>Daily sea level from coastal tide gauge station Woods_Hole in 1978 (Research

					quality database)

				</title>

				<title>Daily sea level from coastal tide gauge station Woods_Hole in 1978 (Research

					quality database)

				</title>

			</titles>

			<publisher>PANGAEA - Data Publisher for Earth &amp; Environmental Science</publisher>

			<publicationYear>2006</publicationYear>

			<subjects>

				<subject subjectScheme="Parameter">DATE/TIME</subject>

				<subject subjectScheme="Parameter">Sea level</subject>

				<subject subjectScheme="Campaign">SeaLevel</subject>

				<subject subjectScheme="Project">World Ocean Circulation Experiment (WOCE)</subject>

			</subjects>

			<contributors>

				<contributor contributorType="HostingInstitution">

					<contributorName>Sea Level Center, University of Hawaii</contributorName>

				</contributor>

			</contributors>

			<dates>

				<date dateType="Collected">1978-01-01T12:00:00/1978-12-31T12:00:00</date>

			</dates>

			<language>eng</language>

			<resourceType resourceTypeGeneral="Dataset">Dataset</resourceType>

			<relatedIdentifiers>

				<relatedIdentifier relatedIdentifierType="URL" relationType="IsDocumentedBy"

				>http://store.pangaea.de/Projects/WOCE/SeaLevel_rqds/Woods_Hole.txt

				</relatedIdentifier>

			</relatedIdentifiers>

			<sizes>

				<size>365 data points</size>

			</sizes>

			<formats>

				<format>text/tab-separated-values</format>

			</formats>

			<rightsList>

				<rights rightsURI="http://creativecommons.org/licenses/by/3.0/">Creative Commons

					Attribution 3.0 Unported (CC-BY)

				</rights>

			</rightsList>

			<geoLocations>

				<geoLocation>

					<geoLocationPoint>41.5233 -70.6717</geoLocationPoint>

				</geoLocation>

			</geoLocations>

		</resource>

	</metadata>

</oai:record>

<?xml version="1.0" encoding="UTF-8"?>

<record xmlns:oaf="http://namespace.openaire.eu/oaf"

        xmlns:dri="http://www.driver-repository.eu/namespace/dri">

	<oai:header xmlns:oai="http://www.openarchives.org/OAI/2.0/"

	>

		<dri:objIdentifier>dli::elsevier::0000d9de7b24b7cb315852926b88e473</dri:objIdentifier>

		<dri:recordIdentifier>10.1016/j.jmgm.2014.05.002::3CQZ</dri:recordIdentifier>

		<dri:dateOfCollection>2016-09-29T16:23:34.316+02:00</dri:dateOfCollection>

		<dri:repositoryId>2d1245d4-c169-4247-9106-0f69f8d752eb_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=</dri:repositoryId>

		<dri:datasourceprefix>dli::elsevier</dri:datasourceprefix>

	</oai:header>

	<metadata>

		<scholix:link xmlns:scholix="http://www.scholix.org">

			<assertion_info>

				<source>dli::elsevier</source>

				<relationType scheme="datacite">unknown</relationType>

			</assertion_info>

			<source>

				<pid type="DOI">10.1016/j.jmgm.2014.05.002</pid>

				<type>publication</type>

			</source>

			<target>

				<pid type="pdb">3CQZ</pid>

				<type>dataset</type>

			</target>

		</scholix:link>

	</metadata>

	<oaf:datainfo>

		<oaf:completionStatus>incomplete</oaf:completionStatus>

		<oaf:provisionMode>collected</oaf:provisionMode>

	</oaf:datainfo>

</record>

package eu.dnetlib.resolver;

import java.util.List;

import com.ximpleware.AutoPilot;

import com.ximpleware.VTDGen;

import com.ximpleware.VTDNav;

import eu.dnetlib.resolver.model.ObjectType;

import eu.dnetlib.resolver.model.ResolvedObject;

import eu.dnetlib.resolver.parser.UtilityParser;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * Created by sandro on 9/26/16.

 */

public class NCBINParser {

	private static final Log log = LogFactory.getLog(NCBINParser.class);

	public ResolvedObject parseRecord(final String record) {

		try {

			final ResolvedObject parsedObject = new ResolvedObject();

			final VTDGen vg = new VTDGen();

			vg.setDoc(record.getBytes());

			vg.parse(true);

			final VTDNav vn = vg.getNav();

			final AutoPilot ap = new AutoPilot(vn);

			final List<String> titles =

					UtilityParser.getTextValue(ap, vn, "//GBSeq_definition");

			parsedObject.setTitles(titles);

			titles.forEach(it -> System.out.println("it = " + it));

			final List<String> descriptions =

					UtilityParser.getTextValue(ap, vn, "//GBSeq_comment");

			if (descriptions != null && descriptions.size() > 0) {

				parsedObject.setDescription(descriptions.get(0));

			}

			parsedObject.setType(ObjectType.dataset);

			return parsedObject;

		} catch (Throwable e) {

			log.error(String.format("Error on parsing document %s", record), e);

			return null;

		}

	}

}

package eu.dnetlib.resolver;

import java.util.Arrays;

import eu.dnetlib.resolver.model.CompletionStatus;

import eu.dnetlib.resolver.model.ObjectProvenance;

import eu.dnetlib.resolver.model.ObjectProvisionMode;

import eu.dnetlib.resolver.model.ResolvedObject;

import org.springframework.beans.factory.annotation.Autowired;

/**

 * Created by sandro on 9/26/16.

 */

public class NCBINResolver extends AbstractPIDResolver {

	private static String baseUrl =

			"https://www.ncbi.nlm.nih.gov/sviewer/viewer.fcgi?db=nuccore&val=%s&page_size=5&fmt_mask=0&report=gbx&retmode=text&page=1&page_size=1";

	@Autowired

	private NCBINParser NCBINParser;

	@Override

	protected boolean canResolvePid(final String pidType) {

		return pidType != null && (pidType.toLowerCase().equals("protein") || pidType.toLowerCase().equals("ncbi-n"));

	}

	@Override

	protected ResolvedObject resolve(final String pid, final String pidType) {

		if ((pid == null))

			return null;

		final ResolvedObject record = NCBINParser.parseRecord(requestURL(String.format(baseUrl, pid)));

		if (record != null) {

			record.setPid(pid);

			record.setPidType(pidType);

			record.setCompletionStatus(CompletionStatus.complete.toString());

			ObjectProvenance provenance = new ObjectProvenance();

			provenance.setCompletionStatus(CompletionStatus.complete.toString());

			provenance.setDatasourceId("dli::r3d100010778");

			provenance.setDatasource("NCBI Nucleotide");

			provenance.setProvisionMode(ObjectProvisionMode.resolved.toString());

			record.setDatasourceProvenance(Arrays.asList(provenance));

		}

		return record;

	}

	public NCBINParser getNCBINParser() {

		return NCBINParser;

	}

	public void setNCBINParser(final NCBINParser NCBINParser) {

		this.NCBINParser = NCBINParser;

	}

}

package eu.dnetlib.resolver.parser;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import com.ximpleware.AutoPilot;

import com.ximpleware.VTDNav;

/**

 * Created by sandro on 9/29/16.

 */

public class UtilityParser {

	public static List<Node> getTextValuesWithAttributes(final AutoPilot ap, final VTDNav vn, final String xpath, final List<String> attributes)

			throws Exception {

		List<Node> results = new ArrayList<>();

		ap.selectXPath(xpath);

		while (ap.evalXPath() != -1) {

			final Node currentNode = new Node();

			final Map<String, String> currentAttributes = new HashMap<>();

			int t = vn.getText();

			currentNode.setTextValue(vn.toNormalizedString(t));

			attributes.forEach(attributeKey -> {

				try {

					int attr = vn.getAttrVal(attributeKey);

					currentAttributes.put(attributeKey, vn.toNormalizedString(attr));

				} catch (Throwable e) {

				}

			});

			currentNode.setAttributes(currentAttributes);

			results.add(currentNode);

		}

		return results;

	}

	public static List<String> getTextValue(final AutoPilot ap, final VTDNav vn, final String xpath) throws Exception {

		List<String> results = new ArrayList<>();

		ap.selectXPath(xpath);

		while (ap.evalXPath() != -1) {

			int t = vn.getText();

			results.add(vn.toNormalizedString(t));

		}

		return results;

	}

	public static String getSingleValue(final AutoPilot ap, final VTDNav nav, final String xpath) throws Exception {

		ap.selectXPath(xpath);

		while (ap.evalXPath() != -1) {

			return nav.toNormalizedString(nav.getText());

		}

		return null;

	}

	public static class Node {

		private String textValue;

		private Map<String, String> attributes;

		public String getTextValue() {

			return textValue;

		}

		public void setTextValue(final String textValue) {

			this.textValue = textValue;

		}

		public Map<String, String> getAttributes() {

			return attributes;

		}

		public void setAttributes(final Map<String, String> attributes) {

			this.attributes = attributes;

		}

	}

}

import java.util.ArrayList;

import java.util.Arrays;

import java.util.List;

import eu.dnetlib.resolver.parser.UtilityParser.Node;

			final String datasourcePrefix = UtilityParser.getSingleValue(ap, vn, "//dri:datasourceprefix");

			final String completionStatus = UtilityParser.getSingleValue(ap, vn, "//oaf:completionStatus");

			final String provisionMode = UtilityParser.getSingleValue(ap, vn, "//oaf:provisionMode");

			final List<Node> identifierType =

					UtilityParser.getTextValuesWithAttributes(ap, vn, "//datacite:identifier", Lists.newArrayList("identifierType"));

				final Node result = identifierType.get(0);

				parsedObject.setPid(result.getTextValue());

				parsedObject.setPidType(result.getAttributes().get("identifierType"));

			final List<Node> relations =

					UtilityParser.getTextValuesWithAttributes(ap, vn, "//datacite:relatedIdentifier", Arrays.asList("relatedIdentifierType", "relationType"));

					final String relationType = relationMap.getAttributes().get("relationType");

					final String relatedIdentifierType = relationMap.getAttributes().get("relatedIdentifierType");

					final String relatedPid = relationMap.getTextValue();

			final List<Node> subjects = UtilityParser.getTextValuesWithAttributes(ap, vn, "//datacite:subject", Arrays.asList("subjectScheme"));

					final SubjectType subject = new SubjectType(subjectMap.getAttributes().get("subjectScheme"), subjectMap.getTextValue());

			final List<String> creators = UtilityParser.getTextValue(ap, vn, "//datacite:creator/datacite:creatorName");

			final List<String> titles = UtilityParser.getTextValue(ap, vn, "//datacite:title");

			final String type = UtilityParser.getSingleValue(ap, vn, "//datacite:resourceType");

			final List<String> dates = UtilityParser.getTextValue(ap, vn, "//datacite:dates/date");

package eu.dnetlib.resolver.parser;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.List;

import com.google.common.collect.Lists;

import com.ximpleware.AutoPilot;

import com.ximpleware.VTDGen;

import com.ximpleware.VTDNav;

import eu.dnetlib.resolver.model.*;

import eu.dnetlib.resolver.parser.UtilityParser.Node;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * Created by sandro on 9/29/16.

 */

public class ScholixResolverParser extends AbstractResolverParser {

	private static final Log log = LogFactory.getLog(ScholixResolverParser.class);

	@Override

	public ResolvedObject parseObject(final String record) {

		try {

			final ResolvedObject parsedObject = new ResolvedObject();

			final VTDGen vg = new VTDGen();

			vg.setDoc(record.getBytes());

			vg.parse(true);

			final VTDNav vn = vg.getNav();

			final AutoPilot ap = new AutoPilot(vn);

			ap.declareXPathNameSpace("dri", "http://www.driver-repository.eu/namespace/dri");

			final String datasourcePrefix = UtilityParser.getSingleValue(ap, vn, "//dri:datasourceprefix");

			ap.declareXPathNameSpace("oaf", "http://namespace.openaire.eu/oaf");

			final String completionStatus = UtilityParser.getSingleValue(ap, vn, "//oaf:completionStatus");

			final String provisionMode = UtilityParser.getSingleValue(ap, vn, "//oaf:provisionMode");

			final ObjectProvenance provenance = new ObjectProvenance();

			provenance.setDatasourceId(datasourcePrefix);

			provenance.setDatasource(datasourcePrefix);

			provenance.setCompletionStatus(completionStatus);

			provenance.setProvisionMode(provisionMode);

			parsedObject.setDatasourceProvenance(Lists.newArrayList(provenance));

			parsedObject.setCompletionStatus(completionStatus);

			ap.declareXPathNameSpace("scholix", "http://www.scholix.org");

			List<Node> sourcePid = UtilityParser.getTextValuesWithAttributes(ap, vn, "//*[local-name()='source']/*[local-name()='pid']", Arrays.asList("type"));

			if (sourcePid == null || sourcePid.size() == 0) {

				return null;

			}

			List<Node> targetPid = UtilityParser.getTextValuesWithAttributes(ap, vn, "//*[local-name()='target']/*[local-name()='pid']", Arrays.asList("type"));

			if (targetPid == null || targetPid.size() == 0) {

				return null;

			}

			parsedObject.setPid(sourcePid.get(0).getTextValue());

			parsedObject.setPidType(sourcePid.get(0).getAttributes().get("type"));

			List<String> sourceTypes = UtilityParser.getTextValue(ap, vn, "//*[local-name()='source']/*[local-name()='type']");

			List<String> targetTypes = UtilityParser.getTextValue(ap, vn, "//*[local-name()='target']/*[local-name()='type']");

			if (sourceTypes != null && sourceTypes.size() > 0) {

				parsedObject.setType(ObjectType.valueOf(sourceTypes.get(0)));

			}

			List<String> relationType = UtilityParser.getTextValue(ap, vn, "//*[local-name()='relationType']");

			String relationSemantic = null;

			if (relationType != null && relationType.size() > 0) {

				relationSemantic = relationType.get(0);

			}

			if (sourceTypes != null && sourceTypes.size() > 0) {

				parsedObject.setType(ObjectType.valueOf(sourceTypes.get(0)));

			}

			final List<ObjectRelation> relations = new ArrayList<>();

			ObjectRelation relation = new ObjectRelation();

			relation.setCompletionStatus(CompletionStatus.incomplete.toString());

			relation.setRelationProvenance(parsedObject.getDatasourceProvenance());

			relation.setRelationSemantics(relationSemantic);

			relation.setTargetPID(new PID(targetPid.get(0).getTextValue(), targetPid.get(0).getAttributes().get("type")));

			if (targetTypes != null || targetTypes.size() > 0)

				relation.setTargetType(ObjectType.valueOf(targetTypes.get(0)));

			relations.add(relation);

			parsedObject.setRelations(relations);

			return parsedObject;

		} catch (Throwable e) {

			log.error("Error on parsing parser");

		}

		return null;

	}

}

	/**

	 * Description of the resolved object

	 */

	/**

	 * A list of subjects of type scheme term

	 */

	 * The related objects.

		String s =

				"DLIObject \n\t Identifier: %s \n\t Pid:%s \n\t PIDType: %s \n\t Entity Type: %s \n\t Titles:%s \n\t Description:%s \n\t Authors:%s, \n\tProvenance%s";

		return String.format(s, this.getIdentifier(), this.getPid(), this.getPidType(), this.getType(), this.getTitles(), this.description,

	<bean id="ncbinResolver"

	      class="eu.dnetlib.resolver.NCBINResolver">

		<property name="cache" ref="dliResolverCache"/>

	</bean>

	<bean id="ncbinParser"

	      class="eu.dnetlib.resolver.NCBINParser"/>

            	<PARAM name="cleanTransformationRuleId"         description="Transformation Rule Identifier"        required="true" managedBy="user"   category="TRANSFORMATION_RULE_ID" type="string" function="listProfiles('TransformationRuleDSResourceType', '//TITLE', 'DLI:')"/>

				<TITLE>DLI: OAI_Datacite to DMF transform</TITLE>

<RESOURCE_PROFILE>

	<HEADER>

		<RESOURCE_IDENTIFIER

				value="d6aa3c16-b6e8-4953-a39e-cb2ea99d2ba8_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU="/>

		<RESOURCE_TYPE value="TransformationRuleDSResourceType"/>

		<RESOURCE_KIND value="TransformationRuleDSResources"/>

		<RESOURCE_URI value=""/>

		<DATE_OF_CREATION value="2014-11-19T11:05:55+01:00"/>

	</HEADER>

	<BODY>

		<CONFIGURATION>

			<IMPORTED/>

			<SCRIPT>

				<TITLE>DLI: Elsevier to Scholix transform</TITLE>

				<CODE><![CDATA[

<xsl:stylesheet version="2.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"

    xmlns:datetime="http://exslt.org/dates-and-times" xmlns:exslt="http://exslt.org/common"

    xmlns:oaf="http://namespace.openaire.eu/oaf"

    xmlns:dri="http://www.driver-repository.eu/namespace/dri"

    exclude-result-prefixes="xsl datetime exslt">

    <xsl:template match="/">

        <record>

            <xsl:copy-of select="//*[local-name()='header']"/>

            <xsl:variable name="source" ><xsl:value-of select="//dri:datasourceprefix"/></xsl:variable>

            <metadata>

                <scholix:link xmlns:scholix="http://www.scholix.org" >

                    <assertion_info>

                        <source><xsl:value-of select="$source"/></source>

                        <relationType scheme="datacite">unknown</relationType>

                    </assertion_info>

                    <source>

                        <pid type="DOI"><xsl:value-of select="//column[./@name='ArticleID']"/></pid>

                        <type>publication</type>

                    </source>

                    <target>

                        <pid>

                            <xsl:attribute name="type"><xsl:value-of select="//column[./@name='db']"/></xsl:attribute>

                            <xsl:value-of select="//column[./@name='datasetID']"/></pid>

                        <type>dataset</type>

                    </target>

                </scholix:link>

            </metadata>

            <oaf:datainfo>

                <oaf:completionStatus>incomplete</oaf:completionStatus>

                <oaf:provisionMode>collected</oaf:provisionMode>

            </oaf:datainfo>

        </record>

    </xsl:template>

</xsl:stylesheet>

]]></CODE>

			</SCRIPT>

		</CONFIGURATION>

		<STATUS/>

		<SECURITY_PARAMETERS/>

	</BODY>

</RESOURCE_PROFILE>

<?xml version="1.0" encoding="UTF-8"?>

<RESOURCE_PROFILE>

	<HEADER>

		<RESOURCE_IDENTIFIER value="8bf9879c-535a-4818-8de7-790a3eb90675_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>

		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>

		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>

		<RESOURCE_URI value=""/>

		<DATE_OF_CREATION value="2001-12-31T12:00:00"/>

	</HEADER>

	<BODY>

		<CONFIGURATION>

			<PARAMETERS>

				<PARAM name="dsId" description="Datasource Id" required="true" type="string"/>

				<PARAM name="interface" description="Datasource Interface" required="true" type="string"/>

				<PARAM name="collMdstoreId" description="Store for collected records" required="true" type="string"/>

				<PARAM name="cleanMdstoreId" description="Store for cleaned records" required="true" type="string"/>

				<PARAM name="patchMdstoreId" description="Store for patched records" required="true" type="string"/>

				<PARAM name="nativeObjectStoreId" description="Object Store for native images" required="true" type="string"/>

				<PARAM name="indexId" description="Index Identifier" required="true" type="string"/>

			</PARAMETERS>

			<WORKFLOW>

				<NODE name="deleteCollMdStore" type="DeleteMDStore" isStart="true">

					<DESCRIPTION>Delete the mdstore of collected records</DESCRIPTION>

					<PARAMETERS>

						<PARAM name="mdstoreId" ref="collMdstoreId"/>

					</PARAMETERS>

					<ARCS>

						<ARC to="deleteCleanMdStore"/>

					</ARCS>

				</NODE>

				<NODE name="deleteCleanMdStore" type="DeleteMDStore">

					<DESCRIPTION>Delete the mdstore of cleaned records</DESCRIPTION>

					<PARAMETERS>

						<PARAM name="mdstoreId" ref="cleanMdstoreId"/>

					</PARAMETERS>

					<ARCS>

						<ARC to="removeApiExtraFields"/>

					</ARCS>

				</NODE>

				<NODE name="removeApiExtraFields" type="RemoveApiExtraFields">

					<DESCRIPTION>Reset the extrafields of the api</DESCRIPTION>

					<PARAMETERS>

						<PARAM name="datasourceId" ref="dsId"/>

						<PARAM name="datasourceInterface" ref="interface"/>

						<PARAM name="fields">

							<LIST>

								<ITEM value="last_collection_total"/>

								<ITEM value="last_collection_date"/>

								<ITEM value="last_collection_mdId"/>

								<ITEM value="last_aggregation_total"/>

								<ITEM value="last_aggregation_date"/>

								<ITEM value="last_aggregation_mdId"/>

							</LIST>

						</PARAM>

					</PARAMETERS>

					<ARCS>

						<ARC to="success"/>

					</ARCS>

				</NODE>

			</WORKFLOW>

		</CONFIGURATION>

	</BODY>

</RESOURCE_PROFILE>

						<PARAM name="description" value="Aggregate  Metadata from DLI Link Provider [Ingestion]"/>

		<!-- https://mvnrepository.com/artifact/org.biojava/biojava3-core -->

		<dependency>

			<groupId>org.biojava</groupId>

			<artifactId>biojava3-core</artifactId>

			<version>3.0</version>

		</dependency>

services.aggregator.country=EU

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

	private String oai_dbName;

		job.getParameters().put("oai_formatName", format);

		job.getParameters().put("oai_formatLayout", layout);

		job.getParameters().put("oai_formatInterpretation", interpretation);

import org.apache.commons.lang3.StringUtils;

	protected String dbName;

	protected String collectionName;

	protected String format, layout, interpretation;

		if (!StringUtils.isBlank(collectionName))

			return collectionName;

		if (!StringUtils.isBlank(format) && !StringUtils.isBlank(layout) && !StringUtils.isBlank(interpretation)) {

			return String.format("%s-%s-%s", format, layout, interpretation);

		}

		return null;

	public String getFormat() {

		return format;

	}

	public void setFormat(final String format) {

		this.format = format;

	}

	public String getLayout() {

		return layout;

	}

	public void setLayout(final String layout) {

		this.layout = layout;

	}

	public String getInterpretation() {

		return interpretation;

	}

	public void setInterpretation(final String interpretation) {

		this.interpretation = interpretation;

	}

import eu.dnetlib.msro.workflows.util.WorkflowsConstants;

import org.antlr.stringtemplate.StringTemplate;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

import org.springframework.core.io.Resource;

			st.setAttribute("xpath", getMetadataIdentifierPath().replace("\"", "'"));

			List<String> namespacePrefix;

			String xQuery = "/*[.//RESOURCE_IDENTIFIER/@value='" + this.datasourceId + "']//EXTRA_FIELDS/FIELD/value[../key='NamespacePrefix']/string()";

			namespacePrefix = this.serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xQuery);

			if (namespacePrefix != null && namespacePrefix.size() != 0) {

				st.setAttribute("namespacePrefix", namespacePrefix.get(0));