D-Net

		assertTrue("different DOIs will NOT drop the score to 0, then evaluate other fields", d == 1.0);

	@Test

	public void testParseDataset2() throws IOException {

		doTest("/eu/dnetlib/data/transform/dataset2.xml", new DatasetToProto());

	}

package eu.dnetlib.data.transform.xml.vtd;

import java.io.IOException;

import java.io.InputStream;

import java.util.Properties;

import com.mongodb.MongoClient;

import com.mongodb.client.MongoDatabase;

import org.springframework.context.annotation.Bean;

import org.springframework.context.annotation.Configuration;

import org.springframework.core.io.ClassPathResource;

@Configuration

public class ConfigurationTestConfig {

	@Bean

	public MongoDatabase db() throws IOException {

		final Properties p = new Properties();

		final ClassPathResource cp = new ClassPathResource("/eu/dnetlib/data/transform/xml/vtd/test.properties");

		try (final InputStream stream = cp.getInputStream()) {

			p.load(stream);

		}

		final MongoClient mongo = new MongoClient(

				p.getProperty("mongodb.host"),

				Integer.valueOf(p.getProperty("mongodb.port")));

		return mongo.getDatabase(p.getProperty("mongodb.dbname"));

	}

}

import java.io.InputStream;

import java.util.List;

        final InputStream resource = this.getClass().getResourceAsStream("/eu/dnetlib/data/transform/publication.xml");

package eu.dnetlib.data.transform.xml.vtd;

import java.io.IOException;

import java.util.Objects;

import java.util.Timer;

import java.util.concurrent.atomic.AtomicInteger;

import java.util.stream.StreamSupport;

import com.google.common.collect.Streams;

import com.mongodb.client.MongoCollection;

import com.mongodb.client.MongoDatabase;

import eu.dnetlib.miscutils.collections.Pair;

import org.apache.commons.lang3.time.StopWatch;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;

import org.bson.Document;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.test.context.ContextConfiguration;

import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import static org.junit.Assert.assertNotNull;

import static org.junit.Assert.assertTrue;

@RunWith(SpringJUnit4ClassRunner.class)

@ContextConfiguration(classes = { ConfigurationTestConfig.class })

public class VtdParserToProtoIT {

	private static final Log log = LogFactory.getLog(VtdParserToProtoIT.class);

	public static final String COLLECTION_NAME = "datacite";

	private static final int BATCH_SIZE = 10000;

	public static final int LOG_FREQ = 5000;

	@Autowired

	private MongoDatabase db;

	@Test

	public void testParseDatacite() throws IOException {

		final MongoCollection<Document> collection = db.getCollection(COLLECTION_NAME);

		final long collectionSize = collection.count();

		log.info(String.format("found %s records in collection '%s'", collectionSize, COLLECTION_NAME));

		final AtomicInteger read = new AtomicInteger(0);

		final DescriptiveStatistics stats = new DescriptiveStatistics();

		final StopWatch timer = new StopWatch();

		final DatasetToProto mapper = new DatasetToProto();

		StreamSupport.stream(collection.find().batchSize(BATCH_SIZE).spliterator(), false)

				.peek(d -> {

					if (read.addAndGet(1) % LOG_FREQ == 0) {

						log.info(String.format("records read so far %s", read.get()));

						log.info(String.format("stats so far %s", stats.toString()));

					}

				})

				.map(d -> (String) d.get("body"))

				.filter(Objects::nonNull)

				.peek(s -> timer.start())

				.map(mapper)

				.forEach(oaf -> {

					assertNotNull(oaf);

					assertTrue(oaf.hasEntity());

					timer.stop();

					stats.addValue(timer.getTime());

					timer.reset();

				});

		log.info(String.format("processed %s/%s records", read.get(), collectionSize));

	}

}

mongodb.host    =   node5.t.openaire.research-infrastructures.eu

mongodb.port    =   27017

mongodb.dbname  =   test_ci

<?xml version="1.0" encoding="UTF-8"?>

<record xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:oaf="http://namespace.openaire.eu/oaf" xmlns:dr="http://www.driver-repository.eu/namespace/dr">

	<oai:header xmlns:oai="http://www.openarchives.org/OAI/2.0/" xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns="http://namespace.openaire.eu/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:prov="http://www.openarchives.org/OAI/2.0/provenance">

		<dri:objIdentifier>datacite____::002bdf21f586e0ea61e992e4724cb8f5</dri:objIdentifier>

		<dri:recordIdentifier>oai:oai.datacite.org:6750738</dri:recordIdentifier>

		<dri:dateOfCollection>2018-03-24T20:16:13.537Z</dri:dateOfCollection>

		<oaf:datasourceprefix>datacite____</oaf:datasourceprefix>

		<identifier xmlns="http://www.openarchives.org/OAI/2.0/">oai:oai.datacite.org:6750738</identifier>

		<datestamp xmlns="http://www.openarchives.org/OAI/2.0/">2018-03-09T23:51:56Z</datestamp>

		<setSpec xmlns="http://www.openarchives.org/OAI/2.0/">CDL</setSpec>

		<setSpec xmlns="http://www.openarchives.org/OAI/2.0/">CDL.CULIS</setSpec>

		<dr:dateOfTransformation>2018-05-12T21:23:38.227Z</dr:dateOfTransformation>

	</oai:header>

	<metadata>

		<resource xmlns="http://datacite.org/schema/kernel-3" xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:oai="http://www.openarchives.org/OAI/2.0/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:prov="http://www.openarchives.org/OAI/2.0/provenance" xsi:schemaLocation="http://datacite.org/schema/kernel-3 http://schema.datacite.org/meta/kernel-3/metadata.xsd">

			<identifier identifierType="DOI">10.7916/D8BV7FXW</identifier>

			<alternateIdentifiers xmlns="http://www.openarchives.org/OAI/2.0/">

				<alternateIdentifier alternateIdentifierType="URL">http://dx.doi.org/10.7916/D8BV7FXW</alternateIdentifier>

			</alternateIdentifiers>

			<titles>

				<title> Is H+3 cooling ever important in primordial gas?</title>

			</titles>

			<publisher>Columbia University</publisher>

			<publicationYear>2009</publicationYear>

			<dates>

				<date dateType="Created">2017-06-27</date>

				<date dateType="Updated">2018-02-17</date>

			</dates>

			<creators>

				<creator>

					<creatorName>Glover, S. C. O.</creatorName>

				</creator>

				<creator>

					<creatorName>Savin, Daniel Wolf</creatorName>

				</creator>

			</creators>

			<subjects>

				<subject>Astrophysics</subject>

				<subject>Plasma (Ionized gases)</subject>

				<subject>Physics</subject>

				<subject>Microphysics</subject>

			</subjects>

			<resourceType resourceTypeGeneral="Dataset">Data</resourceType>

			<descriptions>

				<description descriptionType="Abstract">Full tables of " Is H+3 cooling ever important in primordial gas?".</description>

			</descriptions>

			<relatedIdentifiers>

				<relatedIdentifier relatedIdentifierType="ISSN" relationType="IsPartOf">1365-2966</relatedIdentifier>

				<relatedIdentifier relatedIdentifierType="ISBN" relationType="IsPartOf"/>

				<relatedIdentifier relatedIdentifierType="DOI" relationType="IsVariantFormOf">10.1111/j.1365-2966.2008.14156.x</relatedIdentifier>

			</relatedIdentifiers>

		</resource>

		<dr:CobjCategory>0021</dr:CobjCategory>

		<oaf:dateAccepted>2009-01-01</oaf:dateAccepted>

		<oaf:accessrights>UNKNOWN</oaf:accessrights>

		<oaf:language>und</oaf:language>

		<oaf:hostedBy name="Unknown Repository" id="openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18"/>

		<oaf:collectedFrom name="Datacite" id="openaire____::datacite"/>

	</metadata>

	<about xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:oai="http://www.openarchives.org/OAI/2.0/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:prov="http://www.openarchives.org/OAI/2.0/provenance">

		<provenance xmlns="http://www.openarchives.org/OAI/2.0/provenance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/provenance http://www.openarchives.org/OAI/2.0/provenance.xsd">

			<originDescription harvestDate="2018-03-24T20:16:13.537Z" altered="true">

				<baseURL>https://oai.datacite.org/oai</baseURL>

				<identifier>oai:oai.datacite.org:6750738</identifier>

				<datestamp>2018-03-09T23:51:56Z</datestamp>

				<metadataNamespace/>

			</originDescription>

		</provenance>

		<oaf:datainfo>

			<oaf:inferred>false</oaf:inferred>

			<oaf:deletedbyinference>false</oaf:deletedbyinference>

			<oaf:trust>0.9</oaf:trust>

			<oaf:inferenceprovenance/>

			<oaf:provenanceaction schemename="dnet:provenanceActions" schemeid="dnet:provenanceActions" classname="sysimport:crosswalk:datasetarchive" classid="sysimport:crosswalk:datasetarchive"/>

		</oaf:datainfo>

	</about>

</record>

	public static DatasetToProto newInstance() {

		return new DatasetToProto();

	}

	public VtdException(final String msg, final Exception e) {

		super(msg, e);

	}

	public VtdException(final String msg, final Throwable e) {

		super(msg, e);

	}

	public static final String RELATED_DATASET = "relatedDataSet";

	public static final String RELATED_PUBLICATION = "relatedPublication";

	public static final String RELATED_IDENTIFIER = "relatedIdentifier";

			log.error(xml);

				try {

					addField(md.getBuilder(), d.findFieldByName(fieldName), function.apply(getNodes(ap, vn, xpath)));

				} catch (Throwable e) {

					throw new VtdException(String.format("Error mapping field '%s' from xpath '%s' for record '%s'", fieldName, xpath, objIdentifier), e);

				}

		switch (node.getName()) {

			if (StringUtils.isBlank(node.getTextValue())) {

				return null;

			}

			if (StringUtils.isBlank(a.get("id"))) {

				return null;

			}

			if (StringUtils.isBlank(node.getTextValue())) {

				return null;

			}

		final String name = vn.toRawString(vn.getCurrentIndex());

		currentNode.setName(name.contains(NS_SEPARATOR) ? StringUtils.substringAfter(name, NS_SEPARATOR) : name);

	<build>

		<plugins>

			<plugin>

				<groupId>org.apache.maven.plugins</groupId>

				<artifactId>maven-failsafe-plugin</artifactId>

				<version>2.19.1</version>

				<executions>

					<execution>

						<id>integration-test</id>

						<goals>

							<goal>integration-test</goal>

						</goals>

					</execution>

					<execution>

						<id>verify</id>

						<goals>

							<goal>verify</goal>

						</goals>

					</execution>

				</executions>

			</plugin>

		</plugins>

	</build>

			<groupId>org.apache.commons</groupId>

			<artifactId>commons-lang3</artifactId>

			<version>3.5</version>

		</dependency>

		<!-- test deps -->

		<dependency>

			<groupId>org.mongodb</groupId>

			<artifactId>mongo-java-driver</artifactId>

			<version>${mongodb.driver.version}</version>

			<scope>test</scope>

		<dependency>

			<groupId>org.springframework</groupId>

			<artifactId>spring-context</artifactId>

			<version>${spring.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.springframework</groupId>

			<artifactId>spring-core</artifactId>

			<version>${spring.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.springframework</groupId>

			<artifactId>spring-test</artifactId>

			<version>${spring.version}</version>

			<scope>test</scope>

		</dependency>