D-Net

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-index-client/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-index-client"}

package eu.dnetlib.functionality.index.solr.utils;

import eu.dnetlib.miscutils.functional.UnaryFunction;

import org.apache.oro.text.perl.Perl5Util;

public class HighlightUtils implements UnaryFunction<String, String> {

	public final static String DEFAULT_HL_PRE = "[hl]";

	public final static String DEFAULT_HL_POST = "[/hl]";

	private static String CLEAN_HEADER = "s#\\[/?hl\\]##gm";

	private static String CLEAN_REGEX_OPEN = "<([^>]*)\\[hl\\]([^>]*)>";

	private static String CLEAN_REGEX_CLOSE = "<([^>]*)\\[\\/hl\\]([^>]*)>";

	// private static String CLEAN_REGEX_OPEN = "s#<([^>]*)\\[hl\\]([^>]*)>#<$1$2>#gm";

	// private static String CLEAN_REGEX_CLOSE = "s#<([^>]*)\\[\\/hl\\]([^>]*)>#<$1$2>#gm";

	private Perl5Util p5util = new Perl5Util();

	@Override

	public String evaluate(final String doc) {

		String[] chunk = doc.split("</header>");

		String string = cleanHeader(chunk[0]) + "</header>" + cleanBody(chunk[1]);

		return string;

	}

	private String cleanHeader(final String header) {

		return p5util.substitute(CLEAN_HEADER, header);

	}

	// TODO: implement a faster way to do this

	private String cleanBody(final String body) {

		String res = body.replaceAll(CLEAN_REGEX_OPEN, "<$1$2>").replaceAll(CLEAN_REGEX_CLOSE, "<$1$2>");

		if (res.equals(body)) return res;

		return cleanBody(res);

	}

}

package eu.dnetlib.functionality.index.solr.feed;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.Arrays;

import java.util.List;

import javax.xml.stream.XMLStreamException;

import org.apache.solr.common.SolrInputDocument;

import org.dom4j.DocumentException;

/**

 *

 * @author claudio

 *

 */

public abstract class InputDocumentFactory {

	public static final String INDEX_FIELD_PREFIX = "__";

	public static final String DS_VERSION = INDEX_FIELD_PREFIX + "dsversion";

	public static final String DS_ID = INDEX_FIELD_PREFIX + "dsid";

	public static final String RESULT = "result";

	public static final String INDEX_RESULT = INDEX_FIELD_PREFIX + RESULT;

	public static final String INDEX_RECORD_ID = INDEX_FIELD_PREFIX + "indexrecordidentifier";

	private static final String outFormat = new String("yyyy-MM-dd'T'hh:mm:ss'Z'");

	private final static List<String> dateFormats = Arrays.asList("yyyy-MM-dd'T'hh:mm:ss", "yyyy-MM-dd", "dd-MM-yyyy", "dd/MM/yyyy", "yyyy");

	public abstract SolrInputDocument parseDocument(final String version,

			final String inputDocument,

			final String dsId,

			final String resultName) throws XMLStreamException;

	public abstract SolrInputDocument parseDocument(final String version,

			final String inputDocument,

			final String dsId,

			final String resultName,

			final ResultTransformer resultTransformer) throws XMLStreamException;

	/**

	 * method return a solr-compatible string representation of a date

	 *

	 * @param date

	 * @return the parsed date

	 * @throws DocumentException

	 * @throws ParseException

	 */

	public static String getParsedDateField(final String date) {

		for (String formatString : dateFormats) {

			try {

				return new SimpleDateFormat(outFormat).format(new SimpleDateFormat(formatString).parse(date));

			} catch (ParseException e) {}

		}

		throw new IllegalStateException("unable to parse date: " + date);

	}

	public String parseDate(final String date) {

		return getParsedDateField(date);

	}

}

package eu.dnetlib.functionality.index.solr.feed;

import java.io.StringReader;

import java.io.StringWriter;

import java.util.HashMap;

import java.util.Iterator;

import java.util.List;

import javax.xml.stream.*;

import javax.xml.stream.events.Namespace;

import javax.xml.stream.events.StartElement;

import javax.xml.stream.events.XMLEvent;

import com.google.common.collect.Lists;

import eu.dnetlib.functionality.index.solr.feed.ResultTransformer.Mode;

import org.apache.solr.common.SolrInputDocument;

/**

 * Optimized version of the document parser, drop in replacement of InputDocumentFactory.

 *

 * <p>

 * Faster because:

 * </p>

 * <ul>

 * <li>Doesn't create a DOM for the full document</li>

 * <li>Doesn't execute xpaths agains the DOM</li>

 * <li>Quickly serialize the 'result' element directly in a string.</li>

 * <li>Uses less memory: less pressure on GC and allows more threads to process this in parallel</li>

 * </ul>

 *

 * <p>

 * This class is fully reentrant and can be invoked in parallel.

 * </p>

 *

 * @author marko

 *

 */

public class StreamingInputDocumentFactory extends InputDocumentFactory {

	protected static final String DEFAULTDNETRESULT = "dnetResult";

	protected static final String TARGETFIELDS = "targetFields";

	protected static final String INDEX_RECORD_ID_ELEMENT = "indexRecordIdentifier";

	protected static final String ROOT_ELEMENT = "indexRecord";

	protected ThreadLocal<XMLInputFactory> inputFactory = new ThreadLocal<XMLInputFactory>() {

		@Override

		protected XMLInputFactory initialValue() {

			return XMLInputFactory.newInstance();

		}

	};

	protected ThreadLocal<XMLOutputFactory> outputFactory = new ThreadLocal<XMLOutputFactory>() {

		@Override

		protected XMLOutputFactory initialValue() {

			return XMLOutputFactory.newInstance();

		}

	};

	protected ThreadLocal<XMLEventFactory> eventFactory = new ThreadLocal<XMLEventFactory>() {

		@Override

		protected XMLEventFactory initialValue() {

			return XMLEventFactory.newInstance();

		}

	};

	@Override

	public SolrInputDocument parseDocument(final String version, final String inputDocument, final String dsId, final String resultName)

			throws XMLStreamException {

		return parseDocument(version, inputDocument, dsId, resultName, null);

	}

	@Override

	public SolrInputDocument parseDocument(final String version,

			final String inputDocument,

			final String dsId,

			final String resultName,

			final ResultTransformer resultTransformer) {

		final StringWriter results = new StringWriter();

		final List<Namespace> nsList = Lists.newLinkedList();

		try {

			XMLEventReader parser = inputFactory.get().createXMLEventReader(new StringReader(inputDocument));

			final SolrInputDocument indexDocument = new SolrInputDocument(new HashMap<>());

			while (parser.hasNext()) {

				final XMLEvent event = parser.nextEvent();

				if ((event != null) && event.isStartElement()) {

					final String localName = event.asStartElement().getName().getLocalPart();

					if (ROOT_ELEMENT.equals(localName)) {

						nsList.addAll(getNamespaces(event));

					} else if (INDEX_RECORD_ID_ELEMENT.equals(localName)) {

						final XMLEvent text = parser.nextEvent();

						String recordId = getText(text);

						indexDocument.addField(INDEX_RECORD_ID, recordId);

					} else if (TARGETFIELDS.equals(localName)) {

						parseTargetFields(indexDocument, parser);

					} else if (resultName.equals(localName)) {

						if (resultTransformer == null || !(Mode.empty.equals(resultTransformer.getMode()))) {

							copyResult(indexDocument, results, parser, nsList, resultName, resultTransformer);

						}

					}

				}

			}

			if (version != null) {

				indexDocument.addField(DS_VERSION, version);

			}

			if (dsId != null) {

				indexDocument.addField(DS_ID, dsId);

			}

			if (!indexDocument.containsKey(INDEX_RECORD_ID)) {

				indexDocument.clear();

				System.err.println("missing indexrecord id:\n" + inputDocument);

			}

			return indexDocument;

		} catch (XMLStreamException e) {

			return new SolrInputDocument();

		}

	}

	private List<Namespace> getNamespaces(final XMLEvent event) {

		final List<Namespace> res = Lists.newLinkedList();

		@SuppressWarnings("unchecked")

		Iterator<Namespace> nsIter = event.asStartElement().getNamespaces();

		while (nsIter.hasNext()) {

			Namespace ns = nsIter.next();

			res.add(ns);

		}

		return res;

	}

	/**

	 * Parse the targetFields block and add fields to the solr document.

	 *

	 * @param indexDocument

	 * @param parser

	 * @throws XMLStreamException

	 */

	protected void parseTargetFields(final SolrInputDocument indexDocument, final XMLEventReader parser) throws XMLStreamException {

		boolean hasFields = false;

		while (parser.hasNext()) {

			final XMLEvent targetEvent = parser.nextEvent();

			if (targetEvent.isEndElement() && targetEvent.asEndElement().getName().getLocalPart().equals(TARGETFIELDS)) {

				break;

			}

			if (targetEvent.isStartElement()) {

				final String fieldName = targetEvent.asStartElement().getName().getLocalPart();

				final XMLEvent text = parser.nextEvent();

				String data = getText(text);

				addField(indexDocument, fieldName, data);

				hasFields = true;

			}

		}

		if (!hasFields) {

			indexDocument.clear();

		}

	}

	/**

	 * Copy the /indexRecord/result element and children, preserving namespace declarations etc.

	 *

	 * @param indexDocument

	 * @param results

	 * @param parser

	 * @param nsList

	 * @throws XMLStreamException

	 */

	protected void copyResult(final SolrInputDocument indexDocument,

			final StringWriter results,

			final XMLEventReader parser,

			final List<Namespace> nsList,

			final String dnetResult,

			final ResultTransformer resultTransformer) throws XMLStreamException {

		final XMLEventWriter writer = outputFactory.get().createXMLEventWriter(results);

		for (Namespace ns : nsList) {

			eventFactory.get().createNamespace(ns.getPrefix(), ns.getNamespaceURI());

		}

		StartElement newRecord = eventFactory.get().createStartElement("", null, RESULT, null, nsList.iterator());

		// new root record

		writer.add(newRecord);

		// copy the rest as it is

		while (parser.hasNext()) {

			final XMLEvent resultEvent = parser.nextEvent();

			// TODO: replace with depth tracking instead of close tag tracking.

			if (resultEvent.isEndElement() && resultEvent.asEndElement().getName().getLocalPart().equals(dnetResult)) {

				writer.add(eventFactory.get().createEndElement("", null, RESULT));

				break;

			}

			writer.add(resultEvent);

		}

		writer.close();

		if (resultTransformer != null) {

			indexDocument.addField(INDEX_RESULT, resultTransformer.apply(results.toString()));

		} else {

			indexDocument.addField(INDEX_RESULT, results.toString());

		}

	}

	/**

	 * Helper used to add a field to a solr doc. It avoids to add empy fields

	 *

	 * @param indexDocument

	 * @param field

	 * @param value

	 */

	private final void addField(final SolrInputDocument indexDocument, final String field, final String value) {

		String cleaned = value.trim();

		if (!cleaned.isEmpty()) {

			// log.info("\n\n adding field " + field.toLowerCase() + " value: " + cleaned + "\n");

			indexDocument.addField(field.toLowerCase(), cleaned);

		}

	}

	/**

	 * Helper used to get the string from a text element.

	 *

	 * @param text

	 * @return the

	 */

	protected final String getText(final XMLEvent text) {

		if (text.isEndElement()) // log.warn("skipping because isEndOfElement " + text.asEndElement().getName().getLocalPart());

			return "";

		return text.asCharacters().getData();

	}

}

package eu.dnetlib.functionality.index.solr.feed;

import com.google.common.base.Function;

/**

 * Created by claudio on 17/11/15.

 */

public abstract class ResultTransformer implements Function<String, String> {

	public enum Mode {compress, empty, xslt, base64}

	protected Mode mode;

	public ResultTransformer(final Mode mode) {

		this.mode = mode;

	}

	public Mode getMode() {

		return mode;

	}

	public void setMode(final Mode mode) {

		this.mode = mode;

	}

}

package eu.dnetlib.functionality.index.solr.cql;

import java.util.Map;

import eu.dnetlib.functionality.index.model.Any.ValueType;

import eu.dnetlib.miscutils.functional.UnaryFunction;

import org.springframework.beans.factory.annotation.Required;

/**

 * Factory for the SolrTypeBasedCqlValueTransformerMap class objects

 * 

 * @author claudio

 * 

 */

public class SolrTypeBasedCqlValueTransformerMapFactory {

	/**

	 * Map of functions, injected via spring.

	 */

	private Map<String, UnaryFunction<String, String>> transformerMap;

	/**

	 * Method returns a new instance of SolrTypeBasedCqlValueTransformerMap.

	 * 

	 * @param schema

	 * @return the SolrTypeBasedCqlValueTransformerMap

	 */

	public SolrTypeBasedCqlValueTransformerMap getIt(final Map<String, ValueType> schema) {

		return new SolrTypeBasedCqlValueTransformerMap(schema, getTransformerMap());

	}

	@Required

	public void setTransformerMap(Map<String, UnaryFunction<String, String>> transformerMap) {

		this.transformerMap = transformerMap;

	}

	public Map<String, UnaryFunction<String, String>> getTransformerMap() {

		return transformerMap;

	}

}

package eu.dnetlib.functionality.index.solr.cql;

import eu.dnetlib.miscutils.functional.UnaryFunction;

/**

 * Simply and not very roboust normalizer for solr dates. Basically it handles well yyyy-mm-dd and

 * yyyy-mm-ddThh:mm:ssZ

 * 

 * @author marko

 * 

 */

public class SimpleDateValueTransformer implements UnaryFunction<String, String> {

	@Override

	public String evaluate(final String value) {

		if (!value.endsWith("Z"))

			return value + "T00:00:00Z";

		return value;

	}

}

package eu.dnetlib.functionality.index.solr.cql;

import java.util.Map;

import eu.dnetlib.functionality.cql.CqlValueTransformerMap;

import eu.dnetlib.functionality.index.model.Any.ValueType;

import eu.dnetlib.miscutils.functional.IdentityFunction;

import eu.dnetlib.miscutils.functional.UnaryFunction;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.common.SolrException;

/**

 * This class maps the fields in the given index schema with a transformation rule.

 * 

 * @author marko

 * 

 */

public class SolrTypeBasedCqlValueTransformerMap implements CqlValueTransformerMap {

	/**

	 * logger.

	 */

	private static final Log log = LogFactory.getLog(SolrTypeBasedCqlValueTransformerMap.class); // NOPMD by marko on 11/24/08 5:02 PM

	/**

	 * Index schema.

	 */

	private final Map<String, ValueType> schema;

	/**

	 * Map of functions.

	 */

	private final Map<String, UnaryFunction<String, String>> transformerMap;

	/**

	 * Create value transformer map bound to a specific schema

	 * @param schema

	 * @param transformerMap

	 */

	public SolrTypeBasedCqlValueTransformerMap(final Map<String, ValueType> schema, final Map<String, UnaryFunction<String, String>> transformerMap) {

		this.schema = schema;

		this.transformerMap = transformerMap;

	}

	/**

	 * {@inheritDoc}

	 * 

	 * @see CqlValueTransformerMap#transformerFor(String)

	 */

	@Override

	public UnaryFunction<String, String> transformerFor(final String fieldName) {

		try {

			final ValueType field = schema.get(fieldName);

			if (field != null) {

				UnaryFunction<String, String> res = transformerMap.get(field.name());

				if (res != null) {

					return res;

				}

			}

		} catch (SolrException e) {

			log.debug("cannot find field", e);

		}

		return new IdentityFunction<String>();

	}

}

package eu.dnetlib.functionality.index.model;

import java.util.Date;

/**

 * Interface for a value object, used if the any object is of type string, double, boolean, long, date or date time.

 */

public interface Value extends Any {

	/**

	 * @return The string representation

	 */

	String asString();

	/**

	 * @return The double representation, an InvalidValueTypeException is thrown if the value is not a number. If value

	 *         is a string then this is tried to be converted to a Double.

	 */

	Double asDouble();

	/**

	 * @return The long representation, an InvalidValueTypeException is thrown if the value is not a number. If value is

	 *         a string then this is tried to be converted to a Long.

	 */

	Long asLong();

	/**

	 * @return The boolean representation, an InvalidValueTypeException is thrown if the value is not of type boolean.

	 *         If value is a string then this is tried to be converted to a boolean.

	 */

	Boolean asBoolean();

	/**

	 * @return The date representation, an InvalidValueTypeException is thrown if the value is not of type date or

	 *         datetime. If value is a string then this is tried to be converted to a date.

	 */

	Date asDate();

	/**

	 * @return The date time representation, an InvalidValueTypeException is thrown if the value is not of type

	 *         datetime. If value is a string then this is tried to be converted to a datetime.

	 */

	Date asDateTime();

	/**

	 * @return the value object

	 */

	Object getObject();

}

package eu.dnetlib.functionality.index.model.util;

import java.util.Iterator;

import java.util.Map.Entry;

import eu.dnetlib.functionality.index.model.AnyMap;

import eu.dnetlib.functionality.index.model.DataFactory;

import eu.dnetlib.functionality.index.model.InvalidValueTypeException;

import eu.dnetlib.functionality.index.model.Value;

import eu.dnetlib.functionality.index.model.impl.DefaultDataFactoryImpl;

import org.apache.solr.common.SolrDocumentList;

import org.apache.solr.common.util.NamedList;

/**

 * The Class AnySolrUtil.

 */

public class AnySolrUtil extends AnyUtil {

	/**

	 * Convert named list to any map.

	 * 

	 * @param list

	 *            the list

	 * @param map

	 *            the map

	 * @return the any map

	 */

	@SuppressWarnings("unchecked")

	public static AnyMap convertNamedListToAnyMap(final NamedList<Object> list, final AnyMap map) {

		final Iterator<Entry<String, Object>> it = list.iterator();

		while (it.hasNext()) {

			Entry<String, Object> entry = it.next();

			final String key = entry.getKey();

			final Object obj = entry.getValue();

			if (obj instanceof NamedList<?>) {

				final AnyMap subMap = map.getMap(key, true);

				convertNamedListToAnyMap((NamedList<Object>) obj, subMap);

			} else if (obj instanceof SolrDocumentList) {

				SolrDocumentList docList = (SolrDocumentList) obj;

				AnyMap response = DataFactory.DEFAULT.createAnyMap();

				response.put("numFound", docList.getNumFound());

				response.put("start", docList.getStart());

				response.put("maxScore", docList.getMaxScore());

				response.put("docs", objectToAny(obj));

				map.put("response", response);

			} else {

				try {

					final Value value = DataFactory.DEFAULT.autoConvertValue(obj);

					map.put(key, value);

				} catch (InvalidValueTypeException exception) {

					; // skip

				}

			}

		}

		return map;

	}

	/**

	 * Convert named list to any map.

	 * 

	 * @param list

	 *            the list

	 * @return the any map

	 */

	public static AnyMap convertNamedListToAnyMap(final NamedList<Object> list) {

		return convertNamedListToAnyMap(list, DefaultDataFactoryImpl.INSTANCE.createAnyMap());

	}

}

package eu.dnetlib.functionality.index.model.util;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.Collection;

import java.util.Date;

import java.util.HashSet;

import java.util.Iterator;

import java.util.LinkedHashMap;

import java.util.Map;

import java.util.Map.Entry;

import java.util.Properties;

import java.util.Set;

import eu.dnetlib.functionality.index.model.Any;

import eu.dnetlib.functionality.index.model.AnyMap;

import eu.dnetlib.functionality.index.model.AnySeq;

import eu.dnetlib.functionality.index.model.DataFactory;

import eu.dnetlib.functionality.index.model.Value;

import eu.dnetlib.functionality.index.model.impl.DefaultDataFactoryImpl;

/**

 * utility class for handling / conversion of Any objects.

 *

 * Hint: The Any-to-JSON conversion for Date(Time)s is not symmetric. If the Any object contains Date(Time) values, they will be serialized

 * to String values in simple timestamp format. But when parsing JSON to an Any object, Date(Time) strings will not be recognized anymore,

 * but just read as String values.

 */

public class AnyUtil {

	/** Immutable empty AnyMap instance. */

	public static final AnyMap EMPTY_MAP = DefaultDataFactoryImpl.IMMUTABLE_EMPTY_MAP;

	/**

	 * prevent instance creation.

	 */

	protected AnyUtil() {

		// prevent instance creation

	}

	/**

	 * Converts an Any object into a native java object.

	 *

	 * @param any

	 *            the Any object

	 * @return a Pojo

	 */

	public static Object anyToNative(final Any any) {

		if (any.isMap()) {

			final LinkedHashMap<String, Object> map = new LinkedHashMap<String, Object>();

			for (final Iterator<String> kIt = ((AnyMap) any).keySet().iterator(); kIt.hasNext();) {

				final String key = kIt.next();

				map.put(key, anyToNative(((AnyMap) any).get(key)));

			}

			return map;

		} else if (any.isSeq()) {

			final ArrayList<Object> list = new ArrayList<Object>();

			for (final Iterator<Any> aIt = any.iterator(); aIt.hasNext();) {

				final Any a = aIt.next();

				list.add(anyToNative(a));

			}

			return list;

		} else if (any.isString()) return ((Value) any).asString();

		else if (any.isLong()) return ((Value) any).asLong();

		else if (any.isDouble()) return ((Value) any).asDouble();

		else if (any.isBoolean()) return ((Value) any).asBoolean();

		else if (any.isDate()) return ((Value) any).asDate();

		else if (any.isDateTime()) return ((Value) any).asDateTime();

		else return ((Value) any).getObject();

	}

	/**

	 * Converts an object to an Any (recursively). The leaf object(s) must be convertable by {@link DataFactory#autoConvertValue(Object)}.

	 *

	 * @param object

	 *            The object to be converted. Supported (and tested in this order) are

	 *            <ul>

	 *            <li>{@code Map<String, Object>}

	 *            <li>{@code Collections<Object>}

	 *            <li>{@code Object[]}

	 *            <li>Any other object that can be {@link DataFactory#autoConvertValue(Object)}</li>

	 *            </ul>

	 *

	 * @return The converted Any

	 */

	@SuppressWarnings("unchecked")

	public static Any objectToAny(final Object object) {

		Any value = null;

		if (object instanceof Any) return (Any) object;

		else if (object instanceof Map<?, ?>) {

			value = mapToAny((Map<String, Object>) object);

		} else if (object instanceof Collection<?>) {

			value = collectionToAny((Collection<Object>) object);

		} else if (object.getClass().isArray()) {

			Object[] array = (Object[]) object;

			value = collectionToAny(Arrays.asList(array));

		} else {

			value = scalarObjectToAny(object);

		}

		return value;

	}

	/**

	 * Converts a collection to an AnySeq object.

	 *

	 * @param objects

	 *            The list of objects to convert.

	 * @return An AnySeq containing the objects as Any objects.

	 */

	private static AnySeq collectionToAny(final Collection<Object> objects) {

		AnySeq anySeq = null;

		if (objects != null) {

			anySeq = DataFactory.DEFAULT.createAnySeq();

			for (final Object obj : objects) {

				anySeq.add(objectToAny(obj));

			}

		}

		return anySeq;

	}

	/**

	 * Converts a scalar object to a Value object.

	 *

	 * @param obj

	 *            The object to convert.

	 * @return A Value representing the object.

	 */

	private static Any scalarObjectToAny(final Object obj) {

		return DataFactory.DEFAULT.autoConvertValue(obj);

	}

	/**

	 * Converts a map to an AnyMap object.

	 *

	 * @param map

	 *            The map (String to Object) to convert.

	 * @return An AnyMap representing the map with all Objects converted to Any.

	 */

	private static AnyMap mapToAny(final Map<String, Object> map) {

		AnyMap anyMap = null;

		if (map != null) {

			anyMap = DataFactory.DEFAULT.createAnyMap();

			for (final Entry<String, Object> entry : map.entrySet()) {

				anyMap.put(entry.getKey(), objectToAny(entry.getValue()));

			}

		}

		return anyMap;

	}

	/**

	 * get value for given path(list of keys) from AnyMap object. This methods throws no exception, if the path not exists an empty Any is

	 * the result.

	 *

	 * @param any

	 *            the Any object.

	 * @param path

	 *            path to the entry.

	 * @return value associated to the path.

	 */

	public static Any saveGet(final Any any, final String[] path) {

		if (path.length == 0) return DataFactory.DEFAULT.createAnyMap();

		try {

			Any current = any;

			for (final String key : path) {

				if (current.isMap()) {

					current = ((AnyMap) current).get(key);

				} else {

					current = null;

				}

			}

			if (current == null) return DataFactory.DEFAULT.createStringValue("undef");

			else return current;

		} catch (final Exception e) {

			return DataFactory.DEFAULT.createStringValue("undef");

		}

	}

	/**

	 * convert an exception to an any object.

	 *

	 * @param e

	 *            exception to convert

	 * @return any representation of exception

	 */

	public static AnyMap exceptionToAny(final Throwable e) {

		return exceptionToAny(e, new HashSet<String>());

	}

	/**

	 * convert an exception to an any object. stop in stacktrace printing when hitting known lines again.

	 *

	 * @param e

	 *            exception to convert

	 * @param visitedLines

	 *            lines that have been added to stacktraces before.

	 * @return any representation of exception

	 */

	private static AnyMap exceptionToAny(final Throwable e, final Collection<String> visitedLines) {

		final AnyMap any = DataFactory.DEFAULT.createAnyMap();

		any.put("type", e.getClass().getName());

		if (e.getMessage() != null) {

			any.put("message", e.getMessage());

		}

		final AnySeq st = DataFactory.DEFAULT.createAnySeq();

		for (final StackTraceElement stElement : e.getStackTrace()) {

			final String line = stElement.toString();

			st.add(line);

			if (!visitedLines.add(line)) {

				st.add("...");

				break;

			}

		}

		any.put("at", st);

		if ((e.getCause() != null) && (e.getCause() != e)) {

			any.put("causedBy", exceptionToAny(e.getCause(), visitedLines));

		}

		return any;

	}

	/**

	 * null save version.

	 */

	public static Double asDouble(final Any any) {

		return any == null ? null : any.asValue().asDouble();

	};

	/**

	 * null save version.

	 */

	public static Boolean asBoolean(final Any any) {

		return any == null ? null : any.asValue().asBoolean();

	};

	/**

	 * null save version.

	 */

	public static Date asDateTime(final Any any) {

		return any == null ? null : any.asValue().asDateTime();

	};

	/**

	 * null save version.

	 */

	public static Date asDate(final Any any) {

		return any == null ? null : any.asValue().asDate();

	};

	/**

	 * null save version.

	 */

	public static Long asLong(final Any any) {

		return any == null ? null : any.asValue().asLong();

	};

	/**

	 * null save version.

	 */

	public static String asString(final Any any) {

		return any == null ? null : any.asValue().asString();

	};

	/**

	 * null save version.

	 */

	public static AnyMap asMap(final Any any) {

		return any == null ? null : any.asMap();

	};

	/**

	 * null save version.

	 */

	public static AnySeq asSeq(final Any any) {

		return any == null ? null : any.asSeq();

	};

	/** convert AnyMap to java.util.Properties. */

	public static Properties anyToProperties(final AnyMap anyMap) {

		Properties props = new Properties();

		final Set<String> keySet = anyMap.keySet();

		for (final String key : keySet) {

			props.put(key, anyMap.get(key).toString());

		}

		return props;

	}

	/** convert java.util.Properties to AnyMap. */

	public static AnyMap propertiesToAny(final Properties props) {

		final AnyMap any = DataFactory.DEFAULT.createAnyMap();

		final Set<String> propNames = props.stringPropertyNames();

		for (final String prop : propNames) {

			any.put(prop, props.getProperty(prop));

		}

		return any;

	}

	/**

	 * returns the 1st map in the give SEQ that contains a value with the given name value, or null if not found.

	 *

	 * This is often useful for configs that are contained in a seq such as search filters.

	 *

	 * @since 1.1

	 */

	public static AnyMap findMapInSeq(final AnySeq seq, final String keyName, final String keyValue) {

		for (Any any : seq) {

			final String stringValue = any.asMap().getStringValue(keyName);

			if (keyValue.equals(stringValue)) return any.asMap();

		}

		return null;

	}

}

package eu.dnetlib.functionality.index.model.document;

/**

 * Possible IndexDocument Status.

 */

public enum Status {

	/** The marked. */

	MARKED,

	/** The error. */

	ERROR,

	/** The ok. */

	OK

}

package eu.dnetlib.functionality.index.model.document;

import java.util.Collection;

/**

 * The Interface IndexDocument.

 */

public interface IndexDocument {

	/**

	 * Adds a field with the given name, value and boost. If a field with the name already exists, then the given value is appended to the

	 * value of that field, with the new boost. If the value is a collection, then each of its values will be added to the field.

	 * 

	 * The class type of value and the name parameter should match schema.xml. schema.xml can be found in conf directory under the solr home

	 * by default.

	 * 

	 * @param name

	 *            Name of the field, should match one of the field names defined under "fields" tag in schema.xml.

	 * @param value

	 *            Value of the field, should be of same class type as defined by "type" attribute of the corresponding field in schema.xml.

	 */

	public void addField(String name, Object value);

	/**

	 * Set a field with implied null value for boost.

	 * 

	 * @param name

	 *            name of the field to set

	 * @param value

	 *            value of the field

	 */

	public void setField(String name, Object value);

	/**

	 * Get the first value for a field.

	 * 

	 * @param name

	 *            name of the field to fetch

	 * @return first value of the field or null if not present

	 */

	public Object getFieldValue(String name);

	/**

	 * Gets the field.

	 * 

	 * @param field

	 *            the field

	 * @return the field

	 */

	public IndexField getField(String field);

	/**

	 * Remove a field from the document.

	 * 

	 * @param name

	 *            The field name whose field is to be removed from the document

	 * @return the previous field with <tt>name</tt>, or <tt>null</tt> if there was no field for <tt>key</tt>.

	 */

	public IndexField removeField(String name);

	/**

	 * Get all the values for a field.

	 * 

	 * @param name

	 *            name of the field to fetch

	 * @return value of the field or null if not set

	 */

	public Collection<Object> getFieldValues(String name);

	/**

	 * Get all field names.

	 * 

	 * @return Set of all field names.

	 */

	public Collection<String> getFieldNames();

	/**

	 * return a copy of index Document.

	 * 

	 * @return the index document

	 */

	public IndexDocument deepCopy();

	/**

	 * Gets the status.

	 * 

	 * @return the status

	 */

	public Status getStatus();

	/**

	 * The set status.

	 * 

	 * @param status

	 *            the status

	 * @return the index document

	 */

	public IndexDocument setStatus(Status status);

	/**

	 * If there was an error building the document, it is described here.

	 * 

	 * @return the error

	 */

	public Throwable getError();

	/**

	 * Sets the error.

	 * 

	 * @param error

	 *            the error

	 * @return the index document

	 */

	public IndexDocument setError(final Throwable error);

	/**

	 * Sets the ok status to the index document.

	 * 

	 * @return the index document

	 */

	public IndexDocument setOK();

	/**

	 * Sets the status marked to the index document.

	 * 

	 * @return the index document

	 */

	public IndexDocument setMarked();

}

package eu.dnetlib.functionality.index.model.document;

import java.util.Collection;

import java.util.Iterator;

import java.util.LinkedHashMap;

import java.util.Map;

import java.util.Set;

import eu.dnetlib.functionality.index.model.Any.ValueType;

import eu.dnetlib.functionality.index.utils.IndexFieldUtility;

/**

 * The Class AbstractDocument Represent the field information needed to construct and index Document.

 */

public abstract class AbstractIndexDocument implements Map<String, IndexField>, Iterable<IndexField>, IndexDocument {

	/** The _fields. */

	protected final Map<String, IndexField> fields;

	private static final String DATE_SUFFIX = "\\+\\d{2}:\\d{2}";

	/**

	 * Actual document status.

	 */

	private Status status;

	/**

	 * index schema.

	 */

	protected final Map<String, ValueType> schema;

	/**

	 * If there was an error building the document, it is described here.

	 */

	private Throwable error;

	/**

	 * Instantiates a new abstract document.

	 */

	public AbstractIndexDocument(final Map<String, ValueType> schema, final String dsId) {

		this.schema = schema;

		fields = new LinkedHashMap<String, IndexField>();

		// addField(IndexFieldUtility.DS_ID, dsId);

	}

	/**

	 * Instantiates a new abstract document.

	 *

	 * @param fields

	 *            the fields

	 */

	public AbstractIndexDocument(final Map<String, ValueType> schema, final String dsId, final Map<String, IndexField> fields) {

		this.schema = schema;

		this.fields = fields;

		addField(IndexFieldUtility.DS_ID, dsId);

	}

	/**

	 * Remove all fields and boosts from the document.

	 */

	@Override

	public void clear() {

		if (fields != null) {

			fields.clear();

		}

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.model.document.IndexDocument#addField(java.lang.String, java.lang.Object)

	 */

	@Override

	public void addField(final String name, final Object value) {

		Object hack_value = hackField(name, value);

		IndexField field = fields.get(name);

		if ((field == null) || (field.getValue() == null)) {

			setField(name, hack_value);

		} else {

			field.addValue(hack_value);

		}

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.model.document.IndexDocument#getFieldValue(java.lang.String)

	 */

	@Override

	public Object getFieldValue(final String name) {

		IndexField field = getField(name);

		Object o = null;

		if (field != null) {

			o = field.getFirstValue();

		}

		return o;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.model.document.IndexDocument#getField(java.lang.String)

	 */

	@Override

	public IndexField getField(final String field) {

		return fields.get(field);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.model.document.IndexDocument#getFieldValues(java.lang.String)

	 */

	@Override

	public Collection<Object> getFieldValues(final String name) {

		IndexField field = getField(name);

		if (field != null) return field.getValues();

		return null;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.model.document.IndexDocument#getFieldNames()

	 */

	@Override

	public Collection<String> getFieldNames() {

		return fields.keySet();

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.model.document.IndexDocument#setField(java.lang.String, java.lang.Object)

	 */

	@Override

	public void setField(final String name, final Object value) {

		Object hack_value = hackField(name, value);

		IndexField field = new IndexField(name);

		fields.put(name, field);

		field.setValue(hack_value);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.model.document.IndexDocument#removeField(java.lang.String)

	 */

	@Override

	public IndexField removeField(final String name) {

		return fields.remove(name);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see java.lang.Iterable#iterator()

	 */

	@Override

	public Iterator<IndexField> iterator() {

		return fields.values().iterator();

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see java.lang.Object#toString()

	 */

	@Override

	public String toString() {

		return "IndexDocument: " + fields.values();

	}

	// ---------------------------------------------------

	// MAP interface

	// ---------------------------------------------------

	/**

	 * {@inheritDoc}

	 *

	 * @see java.util.Map#containsKey(java.lang.Object)

	 */

	@Override

	public boolean containsKey(final Object key) {

		return fields.containsKey(key);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see java.util.Map#containsValue(java.lang.Object)

	 */

	@Override

	public boolean containsValue(final Object value) {

		return fields.containsValue(value);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see java.util.Map#entrySet()

	 */

	@Override

	public Set<Entry<String, IndexField>> entrySet() {

		return fields.entrySet();

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see java.util.Map#get(java.lang.Object)

	 */

	@Override

	public IndexField get(final Object key) {

		return fields.get(key);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see java.util.Map#isEmpty()

	 */

	@Override

	public boolean isEmpty() {

		return fields.isEmpty();

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see java.util.Map#keySet()

	 */

	@Override

	public Set<String> keySet() {

		return fields.keySet();

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see java.util.Map#put(java.lang.Object, java.lang.Object)

	 */

	@Override

	public IndexField put(final String key, final IndexField value) {

		return fields.put(key, value);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see java.util.Map#putAll(java.util.Map)

	 */

	@Override

	public void putAll(final Map<? extends String, ? extends IndexField> t) {

		fields.putAll(t);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see java.util.Map#remove(java.lang.Object)

	 */

	@Override

	public IndexField remove(final Object key) {

		return fields.remove(key);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see java.util.Map#size()

	 */

	@Override

	public int size() {

		return fields.size();

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see java.util.Map#values()

	 */

	@Override

	public Collection<IndexField> values() {

		return fields.values();

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.model.document.IndexDocument#getStatus()

	 */

	@Override

	public Status getStatus() {

		return status;

	}

	/**

	 * Sets the status.

	 *

	 * @param status

	 *            the status to set

	 * @return the index document

	 */

	@Override

	public IndexDocument setStatus(final Status status) {

		this.status = status;

		return this;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.model.document.IndexDocument#deepCopy()

	 */

	@Override

	public IndexDocument deepCopy() {

		// TODO Auto-generated method stub

		return null;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.model.document.IndexDocument#getError()

	 */

	@Override

	public Throwable getError() {

		return error;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.model.document.IndexDocument#setOK()