D-Net

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-index-solr-common/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "dnet-index-solr-common"}

package eu.dnetlib.functionality.index.solr.feed;

import java.io.StringReader;

import java.io.StringWriter;

import java.util.Iterator;

import java.util.List;

import javax.xml.stream.*;

import javax.xml.stream.events.Namespace;

import javax.xml.stream.events.StartElement;

import javax.xml.stream.events.XMLEvent;

import com.google.common.collect.Lists;

import eu.dnetlib.functionality.index.solr.feed.ResultTransformer.Mode;

import org.apache.solr.common.SolrInputDocument;

/**

 * Optimized version of the document parser, drop in replacement of InputDocumentFactory.

 *

 * <p>

 * Faster because:

 * </p>

 * <ul>

 * <li>Doesn't create a DOM for the full document</li>

 * <li>Doesn't execute xpaths agains the DOM</li>

 * <li>Quickly serialize the 'result' element directly in a string.</li>

 * <li>Uses less memory: less pressure on GC and allows more threads to process this in parallel</li>

 * </ul>

 *

 * <p>

 * This class is fully reentrant and can be invoked in parallel.

 * </p>

 *

 * @author marko

 *

 */

public class StreamingInputDocumentFactory extends InputDocumentFactory {

	protected static final String DEFAULTDNETRESULT = "dnetResult";

	protected static final String TARGETFIELDS = "targetFields";

	protected static final String INDEX_RECORD_ID_ELEMENT = "indexRecordIdentifier";

	protected static final String ROOT_ELEMENT = "indexRecord";

	protected ThreadLocal<XMLInputFactory> inputFactory = new ThreadLocal<XMLInputFactory>() {

		@Override

		protected XMLInputFactory initialValue() {

			return XMLInputFactory.newInstance();

		}

	};

	protected ThreadLocal<XMLOutputFactory> outputFactory = new ThreadLocal<XMLOutputFactory>() {

		@Override

		protected XMLOutputFactory initialValue() {

			return XMLOutputFactory.newInstance();

		}

	};

	protected ThreadLocal<XMLEventFactory> eventFactory = new ThreadLocal<XMLEventFactory>() {

		@Override

		protected XMLEventFactory initialValue() {

			return XMLEventFactory.newInstance();

		}

	};

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.solr.feed.InputDocumentFactory#parseDocument(eu.dnetlib.functionality.index.solr.feed.IndexDocument,

	 * java.lang.String)

	 */

	@Override

	public SolrInputDocument parseDocument(final String version, final String inputDocument, final String dsId, final String resultName)

			throws XMLStreamException {

		return parseDocument(version, inputDocument, dsId, resultName, null);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.solr.feed.InputDocumentFactory#parseDocument(eu.dnetlib.functionality.index.solr.feed.IndexDocument,

	 *      java.lang.String, com.google.common.base.Function)

	 */

	@Override

	public SolrInputDocument parseDocument(final String version,

			final String inputDocument,

			final String dsId,

			final String resultName,

			final ResultTransformer resultTransformer) {

		final StringWriter results = new StringWriter();

		final List<Namespace> nsList = Lists.newLinkedList();

		try {

			XMLEventReader parser = inputFactory.get().createXMLEventReader(new StringReader(inputDocument));

			final SolrInputDocument indexDocument = new SolrInputDocument();

			while (parser.hasNext()) {

				final XMLEvent event = parser.nextEvent();

				if ((event != null) && event.isStartElement()) {

					final String localName = event.asStartElement().getName().getLocalPart();

					if (ROOT_ELEMENT.equals(localName)) {

						nsList.addAll(getNamespaces(event));

					} else if (INDEX_RECORD_ID_ELEMENT.equals(localName)) {

						final XMLEvent text = parser.nextEvent();

						String recordId = getText(text);

						indexDocument.addField(INDEX_RECORD_ID, recordId);

					} else if (TARGETFIELDS.equals(localName)) {

						parseTargetFields(indexDocument, parser);

					} else if (resultName.equals(localName)) {

						if (resultTransformer == null || !(Mode.empty.equals(resultTransformer.getMode()))) {

							copyResult(indexDocument, results, parser, nsList, resultName, resultTransformer);

						}

					}

				}

			}

			if (version != null) {

				indexDocument.addField(DS_VERSION, version);

			}

			if (dsId != null) {

				indexDocument.addField(DS_ID, dsId);

			}

			if (!indexDocument.containsKey(INDEX_RECORD_ID)) {

				indexDocument.clear();

				System.err.println("missing indexrecord id:\n" + inputDocument);

			}

			return indexDocument;

		} catch (XMLStreamException e) {

			return new SolrInputDocument();

		}

	}

	private List<Namespace> getNamespaces(final XMLEvent event) {

		final List<Namespace> res = Lists.newLinkedList();

		@SuppressWarnings("unchecked")

		Iterator<Namespace> nsIter = event.asStartElement().getNamespaces();

		while (nsIter.hasNext()) {

			Namespace ns = nsIter.next();

			res.add(ns);

		}

		return res;

	}

	/**

	 * Parse the targetFields block and add fields to the solr document.

	 *

	 * @param indexDocument

	 * @param parser

	 * @throws XMLStreamException

	 */

	protected void parseTargetFields(final SolrInputDocument indexDocument, final XMLEventReader parser) throws XMLStreamException {

		boolean hasFields = false;

		while (parser.hasNext()) {

			final XMLEvent targetEvent = parser.nextEvent();

			if (targetEvent.isEndElement() && targetEvent.asEndElement().getName().getLocalPart().equals(TARGETFIELDS)) {

				break;

			}

			if (targetEvent.isStartElement()) {

				final String fieldName = targetEvent.asStartElement().getName().getLocalPart();

				final XMLEvent text = parser.nextEvent();

				String data = getText(text);

				addField(indexDocument, fieldName, data);

				hasFields = true;

			}

		}

		if (!hasFields) {

			indexDocument.clear();

		}

	}

	/**

	 * Copy the /indexRecord/result element and children, preserving namespace declarations etc.

	 *

	 * @param indexDocument

	 * @param results

	 * @param parser

	 * @param nsList

	 * @throws XMLStreamException

	 */

	protected void copyResult(final SolrInputDocument indexDocument,

			final StringWriter results,

			final XMLEventReader parser,

			final List<Namespace> nsList,

			final String dnetResult,

			final ResultTransformer resultTransformer) throws XMLStreamException {

		final XMLEventWriter writer = outputFactory.get().createXMLEventWriter(results);

		for (Namespace ns : nsList) {

			eventFactory.get().createNamespace(ns.getPrefix(), ns.getNamespaceURI());

		}

		StartElement newRecord = eventFactory.get().createStartElement("", null, RESULT, null, nsList.iterator());

		// new root record

		writer.add(newRecord);

		// copy the rest as it is

		while (parser.hasNext()) {

			final XMLEvent resultEvent = parser.nextEvent();

			// TODO: replace with depth tracking instead of close tag tracking.

			if (resultEvent.isEndElement() && resultEvent.asEndElement().getName().getLocalPart().equals(dnetResult)) {

				writer.add(eventFactory.get().createEndElement("", null, RESULT));

				break;

			}

			writer.add(resultEvent);

		}

		writer.close();

		if (resultTransformer != null) {

			indexDocument.addField(INDEX_RESULT, resultTransformer.apply(results.toString()));

		} else {

			indexDocument.addField(INDEX_RESULT, results.toString());

		}

	}

	/**

	 * Helper used to add a field to a solr doc. It avoids to add empy fields

	 *

	 * @param indexDocument

	 * @param field

	 * @param value

	 */

	private final void addField(final SolrInputDocument indexDocument, final String field, final String value) {

		String cleaned = value.trim();

		if (!cleaned.isEmpty()) {

			// log.info("\n\n adding field " + field.toLowerCase() + " value: " + cleaned + "\n");

			indexDocument.addField(field.toLowerCase(), cleaned);

		}

	}

	/**

	 * Helper used to get the string from a text element.

	 *

	 * @param text

	 * @return

	 */

	protected final String getText(final XMLEvent text) {

		if (text.isEndElement()) // log.warn("skipping because isEndOfElement " + text.asEndElement().getName().getLocalPart());

			return "";

		return text.asCharacters().getData();

	}

}

package eu.dnetlib.functionality.index.solr.feed;

import com.google.common.base.Function;

/**

 * Created by claudio on 17/11/15.

 */

public abstract class ResultTransformer implements Function<String, String> {

	public enum Mode {compress, empty, xslt}

	protected Mode mode;

	public ResultTransformer(final Mode mode) {

		this.mode = mode;

	}

	public Mode getMode() {

		return mode;

	}

	public void setMode(final Mode mode) {

		this.mode = mode;

	}

}

package eu.dnetlib.functionality.index.solr.feed;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.Arrays;

import java.util.List;

import javax.xml.stream.XMLStreamException;

import org.apache.solr.common.SolrInputDocument;

import org.dom4j.DocumentException;

/**

 *

 * @author claudio

 *

 */

public abstract class InputDocumentFactory {

	public static final String INDEX_FIELD_PREFIX = "__";

	public static final String DS_VERSION = INDEX_FIELD_PREFIX + "dsversion";

	public static final String DS_ID = INDEX_FIELD_PREFIX + "dsid";

	public static final String RESULT = "result";

	public static final String INDEX_RESULT = INDEX_FIELD_PREFIX + RESULT;

	public static final String INDEX_RECORD_ID = INDEX_FIELD_PREFIX + "indexrecordidentifier";

	private static final String outFormat = new String("yyyy-MM-dd'T'hh:mm:ss'Z'");

	private final static List<String> dateFormats = Arrays.asList("yyyy-MM-dd'T'hh:mm:ss", "yyyy-MM-dd", "dd-MM-yyyy", "dd/MM/yyyy", "yyyy");

	public abstract SolrInputDocument parseDocument(final String version,

			final String inputDocument,

			final String dsId,

			final String resultName) throws XMLStreamException;

	public abstract SolrInputDocument parseDocument(final String version,

			final String inputDocument,

			final String dsId,

			final String resultName,

			final ResultTransformer resultTransformer) throws XMLStreamException;

	/**

	 * method return a solr-compatible string representation of a date

	 *

	 * @param date

	 * @return

	 * @throws DocumentException

	 * @throws ParseException

	 */

	public static String getParsedDateField(final String date) {

		for (String formatString : dateFormats) {

			try {

				return new SimpleDateFormat(outFormat).format(new SimpleDateFormat(formatString).parse(date));

			} catch (ParseException e) {}

		}

		throw new IllegalStateException("unable to parse date: " + date);

	}

	public String parseDate(final String date) {

		return getParsedDateField(date);

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet-hadoop-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-index-solr-common</artifactId>

	<version>1.3.1-SNAPSHOT</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-index-solr-common/trunk</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>org.apache.solr</groupId>

			<artifactId>solr-solrj</artifactId>

			<version>${apache.solr.version}</version>

			<exclusions>

				<exclusion>

					<artifactId>wstx-asl</artifactId>

					<groupId>org.codehaus.woodstox</groupId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>dom4j</groupId>

			<artifactId>dom4j</artifactId>

			<version>1.6.1</version>

		</dependency>

		<dependency>

			<groupId>com.google.guava</groupId>

			<artifactId>guava</artifactId>

			<version>${google.guava.version}</version>

		</dependency>

	</dependencies>

</project>

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-index-solr-common/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "dnet-index-solr-common"}

package eu.dnetlib.functionality.index.solr.feed;

import java.io.StringReader;

import java.io.StringWriter;

import java.util.Iterator;

import java.util.List;

import javax.xml.stream.*;

import javax.xml.stream.events.Namespace;

import javax.xml.stream.events.StartElement;

import javax.xml.stream.events.XMLEvent;

import com.google.common.collect.Lists;

import eu.dnetlib.functionality.index.solr.feed.ResultTransformer.Mode;

import org.apache.solr.common.SolrInputDocument;

/**

 * Optimized version of the document parser, drop in replacement of InputDocumentFactory.

 *

 * <p>

 * Faster because:

 * </p>

 * <ul>

 * <li>Doesn't create a DOM for the full document</li>

 * <li>Doesn't execute xpaths agains the DOM</li>

 * <li>Quickly serialize the 'result' element directly in a string.</li>

 * <li>Uses less memory: less pressure on GC and allows more threads to process this in parallel</li>

 * </ul>

 *

 * <p>

 * This class is fully reentrant and can be invoked in parallel.

 * </p>

 *

 * @author marko

 *

 */

public class StreamingInputDocumentFactory extends InputDocumentFactory {

	protected static final String DEFAULTDNETRESULT = "dnetResult";

	protected static final String TARGETFIELDS = "targetFields";

	protected static final String INDEX_RECORD_ID_ELEMENT = "indexRecordIdentifier";

	protected static final String ROOT_ELEMENT = "indexRecord";

	protected ThreadLocal<XMLInputFactory> inputFactory = new ThreadLocal<XMLInputFactory>() {

		@Override

		protected XMLInputFactory initialValue() {

			return XMLInputFactory.newInstance();

		}

	};

	protected ThreadLocal<XMLOutputFactory> outputFactory = new ThreadLocal<XMLOutputFactory>() {

		@Override

		protected XMLOutputFactory initialValue() {

			return XMLOutputFactory.newInstance();

		}

	};

	protected ThreadLocal<XMLEventFactory> eventFactory = new ThreadLocal<XMLEventFactory>() {

		@Override

		protected XMLEventFactory initialValue() {

			return XMLEventFactory.newInstance();

		}

	};

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.solr.feed.InputDocumentFactory#parseDocument(eu.dnetlib.functionality.index.solr.feed.IndexDocument,

	 * java.lang.String)

	 */

	@Override

	public SolrInputDocument parseDocument(final String version, final String inputDocument, final String dsId, final String resultName)

			throws XMLStreamException {

		return parseDocument(version, inputDocument, dsId, resultName, null);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.solr.feed.InputDocumentFactory#parseDocument(eu.dnetlib.functionality.index.solr.feed.IndexDocument,

	 *      java.lang.String, com.google.common.base.Function)

	 */

	@Override

	public SolrInputDocument parseDocument(final String version,

			final String inputDocument,

			final String dsId,

			final String resultName,

			final ResultTransformer resultTransformer) {

		final StringWriter results = new StringWriter();

		final List<Namespace> nsList = Lists.newLinkedList();

		try {

			XMLEventReader parser = inputFactory.get().createXMLEventReader(new StringReader(inputDocument));

			final SolrInputDocument indexDocument = new SolrInputDocument();

			while (parser.hasNext()) {

				final XMLEvent event = parser.nextEvent();

				if ((event != null) && event.isStartElement()) {

					final String localName = event.asStartElement().getName().getLocalPart();

					if (ROOT_ELEMENT.equals(localName)) {

						nsList.addAll(getNamespaces(event));

					} else if (INDEX_RECORD_ID_ELEMENT.equals(localName)) {

						final XMLEvent text = parser.nextEvent();

						String recordId = getText(text);

						indexDocument.addField(INDEX_RECORD_ID, recordId);

					} else if (TARGETFIELDS.equals(localName)) {

						parseTargetFields(indexDocument, parser);

					} else if (resultName.equals(localName)) {

						if (resultTransformer == null || !(Mode.empty.equals(resultTransformer.getMode()))) {

							copyResult(indexDocument, results, parser, nsList, resultName, resultTransformer);

						}

					}

				}

			}

			if (version != null) {

				indexDocument.addField(DS_VERSION, version);

			}

			if (dsId != null) {

				indexDocument.addField(DS_ID, dsId);

			}

			if (!indexDocument.containsKey(INDEX_RECORD_ID)) {

				indexDocument.clear();

				System.err.println("missing indexrecord id:\n" + inputDocument);

			}

			return indexDocument;

		} catch (XMLStreamException e) {

			return new SolrInputDocument();

		}

	}

	private List<Namespace> getNamespaces(final XMLEvent event) {

		final List<Namespace> res = Lists.newLinkedList();

		@SuppressWarnings("unchecked")

		Iterator<Namespace> nsIter = event.asStartElement().getNamespaces();

		while (nsIter.hasNext()) {

			Namespace ns = nsIter.next();

			res.add(ns);

		}

		return res;

	}

	/**

	 * Parse the targetFields block and add fields to the solr document.

	 *

	 * @param indexDocument

	 * @param parser

	 * @throws XMLStreamException

	 */

	protected void parseTargetFields(final SolrInputDocument indexDocument, final XMLEventReader parser) throws XMLStreamException {

		boolean hasFields = false;

		while (parser.hasNext()) {

			final XMLEvent targetEvent = parser.nextEvent();

			if (targetEvent.isEndElement() && targetEvent.asEndElement().getName().getLocalPart().equals(TARGETFIELDS)) {

				break;

			}

			if (targetEvent.isStartElement()) {

				final String fieldName = targetEvent.asStartElement().getName().getLocalPart();

				final XMLEvent text = parser.nextEvent();

				String data = getText(text);

				addField(indexDocument, fieldName, data);

				hasFields = true;

			}

		}

		if (!hasFields) {

			indexDocument.clear();

		}

	}

	/**

	 * Copy the /indexRecord/result element and children, preserving namespace declarations etc.

	 *

	 * @param indexDocument

	 * @param results

	 * @param parser

	 * @param nsList

	 * @throws XMLStreamException

	 */

	protected void copyResult(final SolrInputDocument indexDocument,

			final StringWriter results,

			final XMLEventReader parser,

			final List<Namespace> nsList,

			final String dnetResult,

			final ResultTransformer resultTransformer) throws XMLStreamException {

		final XMLEventWriter writer = outputFactory.get().createXMLEventWriter(results);

		for (Namespace ns : nsList) {

			eventFactory.get().createNamespace(ns.getPrefix(), ns.getNamespaceURI());

		}

		StartElement newRecord = eventFactory.get().createStartElement("", null, RESULT, null, nsList.iterator());

		// new root record

		writer.add(newRecord);

		// copy the rest as it is

		while (parser.hasNext()) {

			final XMLEvent resultEvent = parser.nextEvent();

			// TODO: replace with depth tracking instead of close tag tracking.

			if (resultEvent.isEndElement() && resultEvent.asEndElement().getName().getLocalPart().equals(dnetResult)) {

				writer.add(eventFactory.get().createEndElement("", null, RESULT));

				break;

			}

			writer.add(resultEvent);

		}

		writer.close();

		if (resultTransformer != null) {

			indexDocument.addField(INDEX_RESULT, resultTransformer.apply(results.toString()));

		} else {

			indexDocument.addField(INDEX_RESULT, results.toString());

		}

	}

	/**

	 * Helper used to add a field to a solr doc. It avoids to add empy fields

	 *

	 * @param indexDocument

	 * @param field

	 * @param value

	 */

	private final void addField(final SolrInputDocument indexDocument, final String field, final String value) {

		String cleaned = value.trim();

		if (!cleaned.isEmpty()) {

			// log.info("\n\n adding field " + field.toLowerCase() + " value: " + cleaned + "\n");

			indexDocument.addField(field.toLowerCase(), cleaned);

		}

	}

	/**

	 * Helper used to get the string from a text element.

	 *

	 * @param text

	 * @return

	 */

	protected final String getText(final XMLEvent text) {

		if (text.isEndElement()) // log.warn("skipping because isEndOfElement " + text.asEndElement().getName().getLocalPart());

			return "";

		return text.asCharacters().getData();

	}

}

package eu.dnetlib.functionality.index.solr.feed;

import com.google.common.base.Function;

/**

 * Created by claudio on 17/11/15.

 */

public abstract class ResultTransformer implements Function<String, String> {

	public enum Mode {compress, empty, xslt}

	protected Mode mode;

	public ResultTransformer(final Mode mode) {

		this.mode = mode;

	}

	public Mode getMode() {

		return mode;

	}

	public void setMode(final Mode mode) {

		this.mode = mode;

	}

}

package eu.dnetlib.functionality.index.solr.feed;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.Arrays;

import java.util.List;

import javax.xml.stream.XMLStreamException;

import org.apache.solr.common.SolrInputDocument;

import org.dom4j.DocumentException;

/**

 *

 * @author claudio

 *

 */

public abstract class InputDocumentFactory {

	public static final String INDEX_FIELD_PREFIX = "__";

	public static final String DS_VERSION = INDEX_FIELD_PREFIX + "dsversion";

	public static final String DS_ID = INDEX_FIELD_PREFIX + "dsid";

	public static final String RESULT = "result";

	public static final String INDEX_RESULT = INDEX_FIELD_PREFIX + RESULT;

	public static final String INDEX_RECORD_ID = INDEX_FIELD_PREFIX + "indexrecordidentifier";

	private static final String outFormat = new String("yyyy-MM-dd'T'hh:mm:ss'Z'");

	private final static List<String> dateFormats = Arrays.asList("yyyy-MM-dd'T'hh:mm:ss", "yyyy-MM-dd", "dd-MM-yyyy", "dd/MM/yyyy", "yyyy");

	public abstract SolrInputDocument parseDocument(final String version,

			final String inputDocument,

			final String dsId,

			final String resultName) throws XMLStreamException;

	public abstract SolrInputDocument parseDocument(final String version,

			final String inputDocument,

			final String dsId,

			final String resultName,

			final ResultTransformer resultTransformer) throws XMLStreamException;

	/**

	 * method return a solr-compatible string representation of a date

	 *

	 * @param date

	 * @return

	 * @throws DocumentException

	 * @throws ParseException

	 */

	public static String getParsedDateField(final String date) {

		for (String formatString : dateFormats) {

			try {

				return new SimpleDateFormat(outFormat).format(new SimpleDateFormat(formatString).parse(date));

			} catch (ParseException e) {}

		}

		throw new IllegalStateException("unable to parse date: " + date);

	}

	public String parseDate(final String date) {

		return getParsedDateField(date);

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet-parent</artifactId>

		<version>1.0.0-SNAPSHOT</version>

		<relativePath/>

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-index-solr-common</artifactId>

	<version>3.0.0-SNAPSHOT</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-index-solr-common/trunk</developerConnection>

	</scm>

	<properties>

		<apache.solr.version>5.4.0</apache.solr.version>

	</properties>

	<dependencies>

		<dependency>

			<groupId>org.apache.solr</groupId>

			<artifactId>solr-solrj</artifactId>

			<version>${apache.solr.version}</version>

			<exclusions>

				<exclusion>

					<artifactId>wstx-asl</artifactId>

					<groupId>org.codehaus.woodstox</groupId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>dom4j</groupId>

			<artifactId>dom4j</artifactId>

			<version>1.6.1</version>

		</dependency>

		<dependency>

			<groupId>com.google.guava</groupId>

			<artifactId>guava</artifactId>

			<version>${google.guava.version}</version>

		</dependency>

	</dependencies>

</project>

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-index-solr-common/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "dnet-index-solr-common"}

package eu.dnetlib.functionality.index.solr.feed;

import java.io.StringReader;

import java.io.StringWriter;

import java.util.Iterator;

import java.util.List;

import javax.xml.stream.*;

import javax.xml.stream.events.Namespace;

import javax.xml.stream.events.StartElement;

import javax.xml.stream.events.XMLEvent;

import com.google.common.collect.Lists;

import eu.dnetlib.functionality.index.solr.feed.ResultTransformer.Mode;

import org.apache.solr.common.SolrInputDocument;

/**

 * Optimized version of the document parser, drop in replacement of InputDocumentFactory.

 *

 * <p>

 * Faster because:

 * </p>

 * <ul>

 * <li>Doesn't create a DOM for the full document</li>

 * <li>Doesn't execute xpaths agains the DOM</li>

 * <li>Quickly serialize the 'result' element directly in a string.</li>

 * <li>Uses less memory: less pressure on GC and allows more threads to process this in parallel</li>

 * </ul>

 *

 * <p>

 * This class is fully reentrant and can be invoked in parallel.

 * </p>

 *

 * @author marko

 *

 */

public class StreamingInputDocumentFactory extends InputDocumentFactory {

	protected static final String DEFAULTDNETRESULT = "dnetResult";

	protected static final String TARGETFIELDS = "targetFields";

	protected static final String INDEX_RECORD_ID_ELEMENT = "indexRecordIdentifier";

	protected static final String ROOT_ELEMENT = "indexRecord";

	protected ThreadLocal<XMLInputFactory> inputFactory = new ThreadLocal<XMLInputFactory>() {

		@Override

		protected XMLInputFactory initialValue() {

			return XMLInputFactory.newInstance();

		}

	};

	protected ThreadLocal<XMLOutputFactory> outputFactory = new ThreadLocal<XMLOutputFactory>() {

		@Override

		protected XMLOutputFactory initialValue() {

			return XMLOutputFactory.newInstance();

		}

	};

	protected ThreadLocal<XMLEventFactory> eventFactory = new ThreadLocal<XMLEventFactory>() {

		@Override

		protected XMLEventFactory initialValue() {

			return XMLEventFactory.newInstance();

		}

	};

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.solr.feed.InputDocumentFactory#parseDocument(eu.dnetlib.functionality.index.solr.feed.IndexDocument,

	 * java.lang.String)

	 */

	@Override

	public SolrInputDocument parseDocument(final String version, final String inputDocument, final String dsId, final String resultName)

			throws XMLStreamException {

		return parseDocument(version, inputDocument, dsId, resultName, null);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.solr.feed.InputDocumentFactory#parseDocument(eu.dnetlib.functionality.index.solr.feed.IndexDocument,

	 *      java.lang.String, com.google.common.base.Function)

	 */

	@Override

	public SolrInputDocument parseDocument(final String version,

			final String inputDocument,

			final String dsId,

			final String resultName,

			final ResultTransformer resultTransformer) {

		final StringWriter results = new StringWriter();

		final List<Namespace> nsList = Lists.newLinkedList();

		try {

			XMLEventReader parser = inputFactory.get().createXMLEventReader(new StringReader(inputDocument));

			final SolrInputDocument indexDocument = new SolrInputDocument();

			while (parser.hasNext()) {

				final XMLEvent event = parser.nextEvent();

				if ((event != null) && event.isStartElement()) {

					final String localName = event.asStartElement().getName().getLocalPart();

					if (ROOT_ELEMENT.equals(localName)) {

						nsList.addAll(getNamespaces(event));

					} else if (INDEX_RECORD_ID_ELEMENT.equals(localName)) {

						final XMLEvent text = parser.nextEvent();

						String recordId = getText(text);

						indexDocument.addField(INDEX_RECORD_ID, recordId);

					} else if (TARGETFIELDS.equals(localName)) {

						parseTargetFields(indexDocument, parser);

					} else if (resultName.equals(localName)) {

						if (resultTransformer == null || !(Mode.empty.equals(resultTransformer.getMode()))) {

							copyResult(indexDocument, results, parser, nsList, resultName, resultTransformer);

						}

					}

				}

			}

			if (version != null) {

				indexDocument.addField(DS_VERSION, version);

			}

			if (dsId != null) {

				indexDocument.addField(DS_ID, dsId);

			}

			if (!indexDocument.containsKey(INDEX_RECORD_ID)) {

				indexDocument.clear();

				System.err.println("missing indexrecord id:\n" + inputDocument);

			}

			return indexDocument;

		} catch (XMLStreamException e) {

			return new SolrInputDocument();

		}

	}

	private List<Namespace> getNamespaces(final XMLEvent event) {

		final List<Namespace> res = Lists.newLinkedList();

		@SuppressWarnings("unchecked")

		Iterator<Namespace> nsIter = event.asStartElement().getNamespaces();

		while (nsIter.hasNext()) {

			Namespace ns = nsIter.next();

			res.add(ns);

		}

		return res;

	}

	/**

	 * Parse the targetFields block and add fields to the solr document.

	 *

	 * @param indexDocument

	 * @param parser

	 * @throws XMLStreamException

	 */

	protected void parseTargetFields(final SolrInputDocument indexDocument, final XMLEventReader parser) throws XMLStreamException {

		boolean hasFields = false;

		while (parser.hasNext()) {

			final XMLEvent targetEvent = parser.nextEvent();

			if (targetEvent.isEndElement() && targetEvent.asEndElement().getName().getLocalPart().equals(TARGETFIELDS)) {

				break;

			}

			if (targetEvent.isStartElement()) {

				final String fieldName = targetEvent.asStartElement().getName().getLocalPart();

				final XMLEvent text = parser.nextEvent();

				String data = getText(text);

				addField(indexDocument, fieldName, data);

				hasFields = true;

			}

		}

		if (!hasFields) {

			indexDocument.clear();

		}

	}

	/**

	 * Copy the /indexRecord/result element and children, preserving namespace declarations etc.

	 *

	 * @param indexDocument

	 * @param results

	 * @param parser

	 * @param nsList

	 * @throws XMLStreamException

	 */

	protected void copyResult(final SolrInputDocument indexDocument,

			final StringWriter results,

			final XMLEventReader parser,

			final List<Namespace> nsList,

			final String dnetResult,

			final ResultTransformer resultTransformer) throws XMLStreamException {

		final XMLEventWriter writer = outputFactory.get().createXMLEventWriter(results);

		for (Namespace ns : nsList) {

			eventFactory.get().createNamespace(ns.getPrefix(), ns.getNamespaceURI());

		}

		StartElement newRecord = eventFactory.get().createStartElement("", null, RESULT, null, nsList.iterator());

		// new root record

		writer.add(newRecord);

		// copy the rest as it is

		while (parser.hasNext()) {

			final XMLEvent resultEvent = parser.nextEvent();

			// TODO: replace with depth tracking instead of close tag tracking.

			if (resultEvent.isEndElement() && resultEvent.asEndElement().getName().getLocalPart().equals(dnetResult)) {

				writer.add(eventFactory.get().createEndElement("", null, RESULT));

				break;

			}

			writer.add(resultEvent);

		}

		writer.close();

		if (resultTransformer != null) {

			indexDocument.addField(INDEX_RESULT, resultTransformer.apply(results.toString()));

		} else {

			indexDocument.addField(INDEX_RESULT, results.toString());

		}

	}

	/**

	 * Helper used to add a field to a solr doc. It avoids to add empy fields

	 *

	 * @param indexDocument

	 * @param field

	 * @param value

	 */

	private final void addField(final SolrInputDocument indexDocument, final String field, final String value) {

		String cleaned = value.trim();

		if (!cleaned.isEmpty()) {

			// log.info("\n\n adding field " + field.toLowerCase() + " value: " + cleaned + "\n");

			indexDocument.addField(field.toLowerCase(), cleaned);

		}

	}

	/**

	 * Helper used to get the string from a text element.

	 *

	 * @param text

	 * @return

	 */

	protected final String getText(final XMLEvent text) {

		if (text.isEndElement()) // log.warn("skipping because isEndOfElement " + text.asEndElement().getName().getLocalPart());

			return "";

		return text.asCharacters().getData();

	}

}

package eu.dnetlib.functionality.index.solr.feed;

import com.google.common.base.Function;

/**

 * Created by claudio on 17/11/15.

 */

public abstract class ResultTransformer implements Function<String, String> {

	public enum Mode {compress, empty, xslt}

	protected Mode mode;

	public ResultTransformer(final Mode mode) {

		this.mode = mode;

	}

	public Mode getMode() {

		return mode;

	}

	public void setMode(final Mode mode) {

		this.mode = mode;

	}

}

package eu.dnetlib.functionality.index.solr.feed;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.Arrays;

import java.util.List;

import javax.xml.stream.XMLStreamException;

import org.apache.solr.common.SolrInputDocument;

import org.dom4j.DocumentException;

/**

 *

 * @author claudio

 *

 */

public abstract class InputDocumentFactory {

	public static final String INDEX_FIELD_PREFIX = "__";

	public static final String DS_VERSION = INDEX_FIELD_PREFIX + "dsversion";

	public static final String DS_ID = INDEX_FIELD_PREFIX + "dsid";

	public static final String RESULT = "result";

	public static final String INDEX_RESULT = INDEX_FIELD_PREFIX + RESULT;

	public static final String INDEX_RECORD_ID = INDEX_FIELD_PREFIX + "indexrecordidentifier";

	private static final String outFormat = new String("yyyy-MM-dd'T'hh:mm:ss'Z'");

	private final static List<String> dateFormats = Arrays.asList("yyyy-MM-dd'T'hh:mm:ss", "yyyy-MM-dd", "dd-MM-yyyy", "dd/MM/yyyy", "yyyy");

	public abstract SolrInputDocument parseDocument(final String version,

			final String inputDocument,

			final String dsId,

			final String resultName) throws XMLStreamException;

	public abstract SolrInputDocument parseDocument(final String version,

			final String inputDocument,

			final String dsId,

			final String resultName,

			final ResultTransformer resultTransformer) throws XMLStreamException;

	/**

	 * method return a solr-compatible string representation of a date

	 *

	 * @param date

	 * @return

	 * @throws DocumentException

	 * @throws ParseException

	 */

	public static String getParsedDateField(final String date) {

		for (String formatString : dateFormats) {

			try {

				return new SimpleDateFormat(outFormat).format(new SimpleDateFormat(formatString).parse(date));

			} catch (ParseException e) {}

		}

		throw new IllegalStateException("unable to parse date: " + date);

	}

	public String parseDate(final String date) {

		return getParsedDateField(date);

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet-hadoop-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-index-solr-common</artifactId>

	<version>1.3.0</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-index-solr-common/tags/dnet-index-solr-common-1.3.0</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>org.apache.solr</groupId>

			<artifactId>solr-solrj</artifactId>

			<version>${apache.solr.version}</version>

			<exclusions>

				<exclusion>

					<artifactId>wstx-asl</artifactId>

					<groupId>org.codehaus.woodstox</groupId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>dom4j</groupId>

			<artifactId>dom4j</artifactId>

			<version>1.6.1</version>

		</dependency>

		<dependency>

			<groupId>com.google.guava</groupId>

			<artifactId>guava</artifactId>

			<version>${google.guava.version}</version>

		</dependency>

	</dependencies>

</project>

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-index-solr-common/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "dnet-index-solr-common"}

package eu.dnetlib.functionality.index.solr.feed;

import java.io.StringReader;

import java.io.StringWriter;

import java.util.Iterator;

import java.util.List;

import javax.xml.stream.XMLEventFactory;

import javax.xml.stream.XMLEventReader;

import javax.xml.stream.XMLEventWriter;

import javax.xml.stream.XMLInputFactory;

import javax.xml.stream.XMLOutputFactory;

import javax.xml.stream.XMLStreamException;

import javax.xml.stream.events.Namespace;

import javax.xml.stream.events.StartElement;

import javax.xml.stream.events.XMLEvent;

import org.apache.solr.common.SolrInputDocument;

import com.google.common.collect.Lists;

/**

 * Optimized version of the document parser, drop in replacement of InputDocumentFactory.

 *

 * <p>

 * Faster because:

 * </p>

 * <ul>

 * <li>Doesn't create a DOM for the full document</li>

 * <li>Doesn't execute xpaths agains the DOM</li>

 * <li>Quickly serialize the 'result' element directly in a string.</li>

 * <li>Uses less memory: less pressure on GC and allows more threads to process this in parallel</li>

 * </ul>

 *

 * <p>

 * This class is fully reentrant and can be invoked in parallel.

 * </p>

 *

 * @author marko

 *

 */

public class StreamingInputDocumentFactory extends InputDocumentFactory {

	protected static final String DEFAULTDNETRESULT = "dnetResult";

	protected static final String TARGETFIELDS = "targetFields";

	protected static final String INDEX_RECORD_ID_ELEMENT = "indexRecordIdentifier";

	protected static final String ROOT_ELEMENT = "indexRecord";

	protected ThreadLocal<XMLInputFactory> inputFactory = new ThreadLocal<XMLInputFactory>() {

		@Override

		protected XMLInputFactory initialValue() {

			return XMLInputFactory.newInstance();

		}

	};

	protected ThreadLocal<XMLOutputFactory> outputFactory = new ThreadLocal<XMLOutputFactory>() {

		@Override

		protected XMLOutputFactory initialValue() {

			return XMLOutputFactory.newInstance();

		}

	};

	protected ThreadLocal<XMLEventFactory> eventFactory = new ThreadLocal<XMLEventFactory>() {

		@Override

		protected XMLEventFactory initialValue() {

			return XMLEventFactory.newInstance();

		}

	};

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.solr.feed.InputDocumentFactory#parseDocument(eu.dnetlib.functionality.index.solr.feed.IndexDocument,

	 *      java.lang.String)

	 */

	@Override

	public SolrInputDocument parseDocument(final String version, final String inputDocument, final String dsId, final String resultName) {

		final StringWriter results = new StringWriter();

		final List<Namespace> nsList = Lists.newLinkedList();

		try {

			XMLEventReader parser = inputFactory.get().createXMLEventReader(new StringReader(inputDocument));

			final SolrInputDocument indexDocument = new SolrInputDocument();

			while (parser.hasNext()) {

				final XMLEvent event = parser.nextEvent();

				if ((event != null) && event.isStartElement()) {

					final String localName = event.asStartElement().getName().getLocalPart();

					if (ROOT_ELEMENT.equals(localName)) {

						nsList.addAll(getNamespaces(event));

					} else if (INDEX_RECORD_ID_ELEMENT.equals(localName)) {

						final XMLEvent text = parser.nextEvent();

						String recordId = getText(text);

						indexDocument.addField(INDEX_RECORD_ID, recordId);

					} else if (TARGETFIELDS.equals(localName)) {

						parseTargetFields(indexDocument, parser);

					} else if (resultName.equals(localName)) {

						copyResult(indexDocument, results, parser, nsList, resultName);

					}

				}

			}

			if (version != null) {

				indexDocument.addField(DS_VERSION, version);

			}

			if (dsId != null) {

				indexDocument.addField(DS_ID, dsId);

			}

			if (!indexDocument.containsKey(INDEX_RECORD_ID)) {

				indexDocument.clear();

				System.err.println("missing indexrecord id:\n" + inputDocument);

			}

			return indexDocument;

		} catch (XMLStreamException e) {

			return new SolrInputDocument();

		}

	}

	private List<Namespace> getNamespaces(final XMLEvent event) {

		final List<Namespace> res = Lists.newLinkedList();

		@SuppressWarnings("unchecked")

		Iterator<Namespace> nsIter = event.asStartElement().getNamespaces();

		while (nsIter.hasNext()) {

			Namespace ns = nsIter.next();

			res.add(ns);

		}

		return res;

	}

	/**

	 * Parse the targetFields block and add fields to the solr document.

	 *

	 * @param indexDocument

	 * @param parser

	 * @throws XMLStreamException

	 */

	protected void parseTargetFields(final SolrInputDocument indexDocument, final XMLEventReader parser) throws XMLStreamException {

		boolean hasFields = false;

		while (parser.hasNext()) {

			final XMLEvent targetEvent = parser.nextEvent();

			if (targetEvent.isEndElement() && targetEvent.asEndElement().getName().getLocalPart().equals(TARGETFIELDS)) {

				break;

			}

			if (targetEvent.isStartElement()) {

				final String fieldName = targetEvent.asStartElement().getName().getLocalPart();

				final XMLEvent text = parser.nextEvent();

				String data = getText(text);

				addField(indexDocument, fieldName, data);

				hasFields = true;

			}

		}

		if (!hasFields) {

			indexDocument.clear();

		}

	}

	/**

	 * Copy the /indexRecord/result element and children, preserving namespace declarations etc.

	 *

	 * @param indexDocument