Index: modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4/pom.xml
===================================================================
--- modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4/pom.xml	(nonexistent)
+++ modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4/pom.xml	(revision 55768)
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+	<parent>
+		<groupId>eu.dnetlib</groupId>
+		<artifactId>dnet45-parent</artifactId>
+		<version>1.0.0</version>
+		<relativePath />
+	</parent>
+	<modelVersion>4.0.0</modelVersion>
+	<groupId>eu.dnetlib</groupId>
+	<artifactId>dnet-index-solr-common</artifactId>
+	<version>2.3.4</version>
+	<scm>
+		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4</developerConnection>
+	</scm>
+	<dependencies>
+		<dependency>
+			<groupId>org.apache.solr</groupId>
+			<artifactId>solr-solrj</artifactId>
+			<version>${apache.solr.version}</version>
+			<exclusions>
+				<exclusion>
+					<artifactId>wstx-asl</artifactId>
+					<groupId>org.codehaus.woodstox</groupId>
+				</exclusion>
+				<exclusion>
+					<artifactId>jcl-over-slf4j</artifactId>
+					<groupId>org.slf4j</groupId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+			<groupId>dom4j</groupId>
+			<artifactId>dom4j</artifactId>
+			<version>${dom4j.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>com.google.guava</groupId>
+			<artifactId>guava</artifactId>
+			<version>${google.guava.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>commons-logging</groupId>
+			<artifactId>commons-logging</artifactId>
+			<version>${commons.logging.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-lang3</artifactId>
+			<version>3.5</version>
+		</dependency>
+
+	</dependencies>
+</project>
Index: modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4/src/main/java/eu/dnetlib/functionality/index/solr/feed/StreamingInputDocumentFactory.java
===================================================================
--- modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4/src/main/java/eu/dnetlib/functionality/index/solr/feed/StreamingInputDocumentFactory.java	(nonexistent)
+++ modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4/src/main/java/eu/dnetlib/functionality/index/solr/feed/StreamingInputDocumentFactory.java	(revision 55768)
@@ -0,0 +1,258 @@
+package eu.dnetlib.functionality.index.solr.feed;
+
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import javax.xml.stream.*;
+import javax.xml.stream.events.Namespace;
+import javax.xml.stream.events.StartElement;
+import javax.xml.stream.events.XMLEvent;
+
+import com.google.common.collect.Lists;
+import eu.dnetlib.functionality.index.solr.feed.ResultTransformer.Mode;
+import org.apache.solr.common.SolrInputDocument;
+
+/**
+ * Optimized version of the document parser, drop in replacement of InputDocumentFactory.
+ *
+ * <p>
+ * Faster because:
+ * </p>
+ * <ul>
+ * <li>Doesn't create a DOM for the full document</li>
+ * <li>Doesn't execute xpaths agains the DOM</li>
+ * <li>Quickly serialize the 'result' element directly in a string.</li>
+ * <li>Uses less memory: less pressure on GC and allows more threads to process this in parallel</li>
+ * </ul>
+ *
+ * <p>
+ * This class is fully reentrant and can be invoked in parallel.
+ * </p>
+ *
+ * @author marko
+ *
+ */
+public class StreamingInputDocumentFactory extends InputDocumentFactory {
+
+	protected static final String DEFAULTDNETRESULT = "dnetResult";
+
+	protected static final String TARGETFIELDS = "targetFields";
+
+	protected static final String INDEX_RECORD_ID_ELEMENT = "indexRecordIdentifier";
+
+	protected static final String ROOT_ELEMENT = "indexRecord";
+
+	protected static final int MAX_FIELD_LENGTH = 25000;
+
+	protected ThreadLocal<XMLInputFactory> inputFactory = new ThreadLocal<XMLInputFactory>() {
+
+		@Override
+		protected XMLInputFactory initialValue() {
+			return XMLInputFactory.newInstance();
+		}
+	};
+
+	protected ThreadLocal<XMLOutputFactory> outputFactory = new ThreadLocal<XMLOutputFactory>() {
+
+		@Override
+		protected XMLOutputFactory initialValue() {
+			return XMLOutputFactory.newInstance();
+		}
+	};
+
+	protected ThreadLocal<XMLEventFactory> eventFactory = new ThreadLocal<XMLEventFactory>() {
+
+		@Override
+		protected XMLEventFactory initialValue() {
+			return XMLEventFactory.newInstance();
+		}
+	};
+
+	@Override
+	public SolrInputDocument parseDocument(final String version, final String inputDocument, final String dsId, final String resultName)
+			throws XMLStreamException {
+		return parseDocument(version, inputDocument, dsId, resultName, null);
+	}
+
+	@Override
+	public SolrInputDocument parseDocument(final String version,
+			final String inputDocument,
+			final String dsId,
+			final String resultName,
+			final ResultTransformer resultTransformer) {
+
+		final StringWriter results = new StringWriter();
+		final List<Namespace> nsList = Lists.newLinkedList();
+		try {
+
+			XMLEventReader parser = inputFactory.get().createXMLEventReader(new StringReader(inputDocument));
+
+			final SolrInputDocument indexDocument = new SolrInputDocument(new HashMap<>());
+
+			while (parser.hasNext()) {
+				final XMLEvent event = parser.nextEvent();
+				if ((event != null) && event.isStartElement()) {
+					final String localName = event.asStartElement().getName().getLocalPart();
+
+					if (ROOT_ELEMENT.equals(localName)) {
+						nsList.addAll(getNamespaces(event));
+					} else if (INDEX_RECORD_ID_ELEMENT.equals(localName)) {
+						final XMLEvent text = parser.nextEvent();
+						String recordId = getText(text);
+						indexDocument.addField(INDEX_RECORD_ID, recordId);
+					} else if (TARGETFIELDS.equals(localName)) {
+						parseTargetFields(indexDocument, parser);
+					} else if (resultName.equals(localName)) {
+						if (resultTransformer == null || !(Mode.empty.equals(resultTransformer.getMode()))) {
+							copyResult(indexDocument, results, parser, nsList, resultName, resultTransformer);
+						}
+					}
+				}
+			}
+
+			if (version != null) {
+				indexDocument.addField(DS_VERSION, version);
+			}
+
+			if (dsId != null) {
+				indexDocument.addField(DS_ID, dsId);
+			}
+
+			if (!indexDocument.containsKey(INDEX_RECORD_ID)) {
+				indexDocument.clear();
+				System.err.println("missing indexrecord id:\n" + inputDocument);
+			}
+
+			return indexDocument;
+		} catch (XMLStreamException e) {
+			return new SolrInputDocument();
+		}
+	}
+
+	private List<Namespace> getNamespaces(final XMLEvent event) {
+		final List<Namespace> res = Lists.newLinkedList();
+		@SuppressWarnings("unchecked")
+		Iterator<Namespace> nsIter = event.asStartElement().getNamespaces();
+		while (nsIter.hasNext()) {
+			Namespace ns = nsIter.next();
+			res.add(ns);
+		}
+		return res;
+	}
+
+	/**
+	 * Parse the targetFields block and add fields to the solr document.
+	 *
+	 * @param indexDocument
+	 * @param parser
+	 * @throws XMLStreamException
+	 */
+	protected void parseTargetFields(final SolrInputDocument indexDocument, final XMLEventReader parser) throws XMLStreamException {
+
+		boolean hasFields = false;
+
+		while (parser.hasNext()) {
+			final XMLEvent targetEvent = parser.nextEvent();
+			if (targetEvent.isEndElement() && targetEvent.asEndElement().getName().getLocalPart().equals(TARGETFIELDS)) {
+				break;
+			}
+
+			if (targetEvent.isStartElement()) {
+				final String fieldName = targetEvent.asStartElement().getName().getLocalPart();
+				final XMLEvent text = parser.nextEvent();
+
+				String data = getText(text);
+
+				addField(indexDocument, fieldName, data);
+				hasFields = true;
+			}
+		}
+
+		if (!hasFields) {
+			indexDocument.clear();
+		}
+	}
+
+	/**
+	 * Copy the /indexRecord/result element and children, preserving namespace declarations etc.
+	 *
+	 * @param indexDocument
+	 * @param results
+	 * @param parser
+	 * @param nsList
+	 * @throws XMLStreamException
+	 */
+	protected void copyResult(final SolrInputDocument indexDocument,
+			final StringWriter results,
+			final XMLEventReader parser,
+			final List<Namespace> nsList,
+			final String dnetResult,
+			final ResultTransformer resultTransformer) throws XMLStreamException {
+		final XMLEventWriter writer = outputFactory.get().createXMLEventWriter(results);
+
+		for (Namespace ns : nsList) {
+			eventFactory.get().createNamespace(ns.getPrefix(), ns.getNamespaceURI());
+		}
+
+		StartElement newRecord = eventFactory.get().createStartElement("", null, RESULT, null, nsList.iterator());
+
+		// new root record
+		writer.add(newRecord);
+
+		// copy the rest as it is
+		while (parser.hasNext()) {
+			final XMLEvent resultEvent = parser.nextEvent();
+
+			// TODO: replace with depth tracking instead of close tag tracking.
+			if (resultEvent.isEndElement() && resultEvent.asEndElement().getName().getLocalPart().equals(dnetResult)) {
+				writer.add(eventFactory.get().createEndElement("", null, RESULT));
+				break;
+			}
+
+			writer.add(resultEvent);
+		}
+		writer.close();
+
+		if (resultTransformer != null) {
+			indexDocument.addField(INDEX_RESULT, resultTransformer.apply(results.toString()));
+		} else {
+			indexDocument.addField(INDEX_RESULT, results.toString());
+		}
+	}
+
+	/**
+	 * Helper used to add a field to a solr doc. It avoids to add empy fields
+	 *
+	 * @param indexDocument
+	 * @param field
+	 * @param value
+	 */
+	private final void addField(final SolrInputDocument indexDocument, final String field, final String value) {
+		String cleaned = value.trim();
+		if (!cleaned.isEmpty()) {
+			// log.info("\n\n adding field " + field.toLowerCase() + " value: " + cleaned + "\n");
+			indexDocument.addField(field.toLowerCase(), cleaned);
+		}
+	}
+
+	/**
+	 * Helper used to get the string from a text element.
+	 *
+	 * @param text
+	 * @return the element text value
+	 */
+	protected final String getText(final XMLEvent text) {
+		if (text.isEndElement()) // log.warn("skipping because isEndOfElement " + text.asEndElement().getName().getLocalPart());
+			return "";
+
+		final String data = text.asCharacters().getData();
+		if (data != null && data.length() > MAX_FIELD_LENGTH) {
+			return data.substring(0, MAX_FIELD_LENGTH);
+		}
+
+		return data;
+	}
+
+}
Index: modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4/src/main/java/eu/dnetlib/functionality/index/solr/feed/InputDocumentFactory.java
===================================================================
--- modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4/src/main/java/eu/dnetlib/functionality/index/solr/feed/InputDocumentFactory.java	(nonexistent)
+++ modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4/src/main/java/eu/dnetlib/functionality/index/solr/feed/InputDocumentFactory.java	(revision 55768)
@@ -0,0 +1,67 @@
+package eu.dnetlib.functionality.index.solr.feed;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.List;
+import javax.xml.stream.XMLStreamException;
+
+import org.apache.solr.common.SolrInputDocument;
+import org.dom4j.DocumentException;
+
+/**
+ *
+ * @author claudio
+ *
+ */
+public abstract class InputDocumentFactory {
+
+	public static final String INDEX_FIELD_PREFIX = "__";
+
+	public static final String DS_VERSION = INDEX_FIELD_PREFIX + "dsversion";
+
+	public static final String DS_ID = INDEX_FIELD_PREFIX + "dsid";
+
+	public static final String RESULT = "result";
+
+	public static final String INDEX_RESULT = INDEX_FIELD_PREFIX + RESULT;
+
+	public static final String INDEX_RECORD_ID = INDEX_FIELD_PREFIX + "indexrecordidentifier";
+
+	private static final String outFormat = new String("yyyy-MM-dd'T'hh:mm:ss'Z'");
+
+	private final static List<String> dateFormats = Arrays.asList("yyyy-MM-dd'T'hh:mm:ss", "yyyy-MM-dd", "dd-MM-yyyy", "dd/MM/yyyy", "yyyy");
+
+	public abstract SolrInputDocument parseDocument(final String version,
+			final String inputDocument,
+			final String dsId,
+			final String resultName) throws XMLStreamException;
+
+	public abstract SolrInputDocument parseDocument(final String version,
+			final String inputDocument,
+			final String dsId,
+			final String resultName,
+			final ResultTransformer resultTransformer) throws XMLStreamException;
+
+	/**
+	 * method return a solr-compatible string representation of a date
+	 *
+	 * @param date
+	 * @return the formatted date string
+	 * @throws DocumentException
+	 * @throws ParseException
+	 */
+	public static String getParsedDateField(final String date) {
+		for (String formatString : dateFormats) {
+			try {
+				return new SimpleDateFormat(outFormat).format(new SimpleDateFormat(formatString).parse(date));
+			} catch (ParseException e) {}
+		}
+		throw new IllegalStateException("unable to parse date: " + date);
+	}
+
+	public String parseDate(final String date) {
+		return getParsedDateField(date);
+	}
+
+}
Index: modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4/src/main/java/eu/dnetlib/functionality/index/solr/feed/ResultTransformer.java
===================================================================
--- modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4/src/main/java/eu/dnetlib/functionality/index/solr/feed/ResultTransformer.java	(nonexistent)
+++ modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4/src/main/java/eu/dnetlib/functionality/index/solr/feed/ResultTransformer.java	(revision 55768)
@@ -0,0 +1,26 @@
+package eu.dnetlib.functionality.index.solr.feed;
+
+import com.google.common.base.Function;
+
+/**
+ * Created by claudio on 17/11/15.
+ */
+public abstract class ResultTransformer implements Function<String, String> {
+
+	public enum Mode {compress, empty, xslt, base64}
+
+	protected Mode mode;
+
+	public ResultTransformer(final Mode mode) {
+		this.mode = mode;
+	}
+
+	public Mode getMode() {
+		return mode;
+	}
+
+	public void setMode(final Mode mode) {
+		this.mode = mode;
+	}
+
+}
Index: modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4/src/main/java/eu/dnetlib/functionality/index/utils/ZkServers.java
===================================================================
--- modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4/src/main/java/eu/dnetlib/functionality/index/utils/ZkServers.java	(nonexistent)
+++ modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4/src/main/java/eu/dnetlib/functionality/index/utils/ZkServers.java	(revision 55768)
@@ -0,0 +1,58 @@
+package eu.dnetlib.functionality.index.utils;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.util.List;
+import java.util.Optional;
+
+public class ZkServers {
+
+    private static final Log log = LogFactory.getLog(ZkServers.class);
+    public static final String SEPARATOR = "/";
+
+    private List<String> hosts;
+
+    private Optional<String> chroot;
+
+    public static ZkServers newInstance(final String zkUrl) {
+
+        //quorum0:2182,quorum1:2182,quorum2:2182,quorum3:2182,quorum4:2182/solr-dev-openaire
+        String urls = zkUrl;
+        final Optional<String> chRoot = Optional.of(SEPARATOR + StringUtils.substringAfterLast(zkUrl, SEPARATOR));
+        if (chRoot.isPresent() && StringUtils.isNotBlank(chRoot.get())) {
+            log.debug(String.format("found zk chroot %s", chRoot));
+            urls = zkUrl.replace(chRoot.get(), "");
+        }
+
+        final List<String> urlList = Lists.newArrayList(Splitter.on(",").omitEmptyStrings().split(urls));
+        log.debug(String.format("zk urls %s", zkUrl));
+
+        return new ZkServers(urlList, chRoot);
+    }
+
+    public ZkServers(List<String> hosts, Optional<String> chroot) {
+        this.hosts = hosts;
+        this.chroot = chroot;
+    }
+
+    public List<String> getHosts() {
+        return hosts;
+    }
+
+    public void setHosts(List<String> hosts) {
+        this.hosts = hosts;
+    }
+
+    public Optional<String> getChroot() {
+        return chroot;
+    }
+
+    public void setChroot(Optional<String> chroot) {
+        this.chroot = chroot;
+    }
+}
+
Index: modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4/deploy.info
===================================================================
--- modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4/deploy.info	(nonexistent)
+++ modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4/deploy.info	(revision 55768)
@@ -0,0 +1 @@
+{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-index-solr-common/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-index-solr-common"}
\ No newline at end of file
Index: modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4
===================================================================
--- modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4	(nonexistent)
+++ modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4	(revision 55768)

Property changes on: modules/dnet-index-solr-common/tags/dnet-index-solr-common-2.3.4
___________________________________________________________________
Added: svn:ignore
## -0,0 +1,5 ##
+target
+.settings
+.classpath
+.project
+*.iml
