D-Net

package eu.dnetlib.functionality.index.utils;

import java.io.StringReader;

import java.util.List;

import java.util.Map;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Autowired;

import org.xml.sax.InputSource;

import com.google.common.collect.HashBasedTable;

import com.google.common.collect.Table;

import com.mycila.xmltool.CallBack;

import com.mycila.xmltool.XMLDoc;

import com.mycila.xmltool.XMLTag;

import eu.dnetlib.data.provision.index.rmi.IndexServiceException;

import eu.dnetlib.functionality.index.client.IndexClientException;

public class MDFormatReader {

	static class XmlUtils {

		/**

		 * helper method, parses a list of fields.

		 *

		 * @param fields

		 *            the given fields

		 * @return the parsed fields

		 * @throws IndexServiceException

		 *             if cannot parse the fields

		 */

		public static Document parse(final String xml) {

			try {

				return new SAXReader().read(new StringReader(xml));

			} catch (DocumentException e) {

				throw new IllegalArgumentException("cannot parse: " + xml);

			}

		}

		public static InputSource asInputSource(final String input) throws DocumentException {

			return new InputSource(new StringReader(parse(input).asXML()));

		}

	}

	@Autowired

	private ServiceTools serviceTools;

	@Autowired

	private MetadataReferenceFactory mdFactory;

	public List<MetadataReference> listMDRefs() throws IndexClientException {

		return serviceTools.listMDRefs();

	}

	public Document getFields(final MetadataReference mdRef) {

		String fields = serviceTools.getIndexFields(mdRef);

		return (fields != null) && !fields.isEmpty() ? XmlUtils.parse(fields) : null;

	}

	public Map<String, String> getAttributeMap(final MetadataReference mdRef, final String attribute) throws IndexClientException {

		return getAttributeTable(mdRef, attribute).column(attribute);

	}

	public Table<String, String, String> getAttributeTable(final MetadataReference mdRef, final String... attributeList) throws IndexClientException {

		final String fields = serviceTools.getIndexFields(mdRef);

		if (fields.isEmpty()) throw new IndexClientException("No result getting index layout informations");

		final Table<String, String, String> t = HashBasedTable.create();

		XMLDoc.from(serviceTools.getIndexFields(mdRef), false).forEach("//FIELD", new CallBack() {

			@Override

			public void execute(final XMLTag field) {

				for (String attribute : attributeList) {

					String value = null;

					if ("xpath".equals(attribute)) {

						if (!(field.hasAttribute("xpath") || field.hasAttribute("value"))) return;

						value = field.hasAttribute("xpath") ? field.getAttribute("xpath") : field.getAttribute("value");

					}

					if ("weight".equals(attribute)) {

						if (!(field.hasAttribute(attribute))) return;

						value = field.hasAttribute(attribute) ? field.getAttribute(attribute) : "";

					}

					if (value == null) {

						value = field.getAttribute(attribute);

					}

					t.put(field.getAttribute("name").toLowerCase(), attribute, value);

				}

			}

		});

		return t;

	}

}

package eu.dnetlib.functionality.index.model;

/**

 * A runtime exception that will be thrown if the value types of the Any objects do not match the expected types.

 * 

 */

public class InvalidValueTypeException extends RuntimeException {

	/** exceptions can be serialized. */

	private static final long serialVersionUID = 1L;

	/** unsupported value type that caused this exception. */

	private final Class<?> _unsupportedType;

	/**

	 * Creates an InvalidValueTypeException with the given message.

	 * 

	 * @param message

	 *            The message for the user.

	 */

	public InvalidValueTypeException(final String message) {

		this(message, (Class<?>) null);

	}

	/**

	 * Instantiates a new invalid value type exception.

	 * 

	 * @param message

	 *            the message

	 * @param unsupportedType

	 *            the unsupported type

	 */

	public InvalidValueTypeException(final String message, final Class<?> unsupportedType) {

		super(message);

		_unsupportedType = unsupportedType;

	}

	/**

	 * Instantiates a new invalid value type exception.

	 * 

	 * @param unsupportedType

	 *            the unsupported type

	 */

	public InvalidValueTypeException(final Class<?> unsupportedType) {

		this("type not supported: " + unsupportedType, unsupportedType);

	}

	/**

	 * @param format

	 * @param e

	 */

	public InvalidValueTypeException(final String message, final Exception e) {

		super(message, e);

		_unsupportedType = null;

	}

	/**

	 * Gets the unsupported type. might be null.

	 * 

	 * @return the unsupported type

	 */

	public Class<?> getUnsupportedType() {

		return _unsupportedType;

	}

}

package eu.dnetlib.functionality.index.query;

import java.util.List;

import java.util.Map;

import com.google.common.collect.BiMap;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import eu.dnetlib.data.provision.index.rmi.IndexServiceException;

import eu.dnetlib.functionality.cql.CqlTranslator;

import eu.dnetlib.functionality.cql.lucene.TranslatedQuery;

import eu.dnetlib.functionality.index.client.IndexClient;

import eu.dnetlib.functionality.index.client.IndexClientException;

import eu.dnetlib.functionality.index.query.Pruner.Result;

import eu.dnetlib.functionality.index.utils.MetadataReference;

import org.springframework.beans.factory.annotation.Autowired;

/**

 * A factory for creating IndexQuery objects.

 */

public abstract class IndexQueryFactory {

	/**

	 * Query tree pruner.

	 */

	private Pruner pruner;

	/**

	 * Query tree pruner. Collects parameters which affect the semantic of the cql parser.

	 */

	private Pruner cqlPruner;

	/** The default query params. */

	private Map<String, List<String>> defaultQueryParams;

	/** CqlTranslator. */

	@Autowired

	private CqlTranslator translator;

	/** The browse aliases. */

	@Autowired

	private BrowseAliases browseAliases;

	/** The weights. */

	@Autowired

	private Weights weights;

	/**

	 * New instance.

	 * 

	 * @param cql

	 *            the cql

	 * @param res

	 *            the res

	 * @param queryLanguage

	 *            the query language

	 * @return the index query

	 */

	protected abstract IndexQuery newInstance(final TranslatedQuery cql, final Result res, final QueryLanguage queryLanguage);

	/**

	 * Sets the query options.

	 * 

	 * @param indexQuery

	 *            the index query

	 * @return the index query

	 */

	protected abstract IndexQuery setQueryOptions(final IndexQuery indexQuery, final IndexClient client);

	/**

	 * Gets the index query.

	 * 

	 * @param lang

	 *            the lang

	 * @param query

	 *            the query

	 * @param mdRef

	 *            the md ref

	 * @return the index query

	 * @throws IndexServiceException

	 *             the index service exception

	 */

	public IndexQuery getIndexQuery(final QueryLanguage lang, final String query, final IndexClient client, final MetadataReference mdRef)

			throws IndexClientException {

		String myquery = query;

		if ((myquery == null) || myquery.isEmpty()) throw new IndexClientException("query cannot be empty or null");

		try {

			final Result cqlRes = getCqlPruner().prune(getCqlPruner().parse(myquery));

			final Result res = getPruner().prune(cqlRes.getNode());

			final TranslatedQuery tQuery = translator.getTranslatedQuery(res.getNode(), client.getCqlValueTransformerMap(mdRef),

					overrideCqlParams(cqlRes.getOptionMap()), browseAliases.get(mdRef), weights.get(mdRef));

			return setQueryOptions(newInstance(tQuery, res, lang), client);

		} catch (Exception e) {

			throw new IndexClientException(e);

		}

	}

	/**

	 * Method overrides the default values in the defaultQueryParams with the given override map.

	 * 

	 * @param override

	 *            the map containing the override values.

	 * @return the overridden parameter map

	 * 

	 */

	private Map<String, List<String>> overrideCqlParams(final Map<String, List<String>> override) {

		Map<String, List<String>> cqlParams = Maps.newHashMap();

		cqlParams.putAll(getDefaultQueryParams());

		cqlParams.putAll(override);

		return cqlParams;

	}

	public List<String> getBrowsableFields(final List<String> fields, final MetadataReference mdRef) throws IndexClientException {

		return getBrowsableFields(fields, browseAliases.get(mdRef));

	}

	/**

	 * Gets the list of aliases available for browse

	 * 

	 * @param fields

	 *            list of input fields

	 * @param aliases

	 *            key= non-browasbale-field-name, value=browsable-field-name

	 * @return the list of browasable field names

	 */

	public List<String> getBrowsableFields(final List<String> fields, final BiMap<String, String> aliases) {

		List<String> browsables = Lists.newArrayListWithExpectedSize(fields.size());

		for (String f : fields) {

			if (aliases.containsKey(f)) {

				browsables.add(aliases.get(f));

			} else {

				browsables.add(f);

			}

		}

		return browsables;

	}

	/**

	 * Gets the pruner.

	 * 

	 * @return the pruner

	 */

	public Pruner getPruner() {

		return pruner;

	}

	/**

	 * Sets the pruner.

	 * 

	 * @param pruner

	 *            the new pruner

	 */

	public void setPruner(final Pruner pruner) {

		this.pruner = pruner;

	}

	/**

	 * Gets the cql pruner.

	 * 

	 * @return the cql pruner

	 */

	public Pruner getCqlPruner() {

		return cqlPruner;

	}

	/**

	 * Sets the cql pruner.

	 * 

	 * @param cqlPruner

	 *            the new cql pruner

	 */

	public void setCqlPruner(final Pruner cqlPruner) {

		this.cqlPruner = cqlPruner;

	}

	/**

	 * Gets the default query params.

	 * 

	 * @return the default query params

	 */

	public Map<String, List<String>> getDefaultQueryParams() {

		return defaultQueryParams;

	}

	/**

	 * Sets the default query params.

	 * 

	 * @param defaultQueryParams

	 *            the default query params

	 */

	public void setDefaultQueryParams(final Map<String, List<String>> defaultQueryParams) {

		this.defaultQueryParams = defaultQueryParams;

	}

}

package eu.dnetlib.functionality.index.model;

import java.text.DateFormat;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.Date;

import java.util.TimeZone;

import java.util.regex.Pattern;

/**

 * helper class for formatting and parsing Values. all methods synchronize on the used local formatter object, so you

 * can use the shared instance. Using multiple instances may improve performance, though, because of less

 * synchronization.

 */

public class ValueFormatHelper {

	/** shared global helper instance. */

	public static final ValueFormatHelper INSTANCE = new ValueFormatHelper();

	/** The max. length of strings to be parsed as date. */

	private static final int DATE_LENGTH = 10;

	/** The length of strings to be parsed as date time for (default) pattern 1. */

	private static final int DATE_TIME_LENGTH_PATTERN_DEFAULT = 28;

	/** The length of strings to be parsed as date time for pattern 2 and 3. */

	private static final int DATE_TIME_LENGTH_PATTERN_2_AND_3 = 24;

	/** The length of strings to be parsed as date time for pattern 4. */

	private static final int DATE_TIME_LENGTH_PATTERN_4 = 20;

	/** formatter to create and parse standard string representations of Date values: "yyyy-MM-dd". */

	private final DateFormat _formatDate = new SimpleDateFormat("yyyy-MM-dd");

	/** valid Datetime value pattern with milliseconds and time zone (default for printing). */

	private final DateFormat _formatDateTimePatternDefault = getDefaultDateTimeFormat();

	/** valid Datetime value pattern with time zone. */

	private final DateFormat _formatDateTimePattern2 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");

	/** valid Datetime value pattern with milliseconds and time zone 'Z' (UTC). */

	private final DateFormat _formatDateTimePattern3 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");

	/** valid Datetime value pattern without milliseconds and time zone 'Z' (UTC). */

	private final DateFormat _formatDateTimePattern4 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");

	/** pattern for checking if the string might be a date. */

	private final Pattern _formatPatternDateDefault = Pattern.compile("\\d{4}-\\d{2}-\\d{2}");

	/** pattern for checking if the string might be a date time for default pattern. */

	private final Pattern _formatPatternTimeDefault = Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}[+-]{1}\\d{4}");

	/** pattern for checking if the string might be a date time for pattern 2. */

	private final Pattern _formatPatternTime2 = Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}[+-]{1}\\d{4}");

	/** pattern for checking if the string might be a date time for pattern 3. */

	private final Pattern _formatPatternTime3 = Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z");

	/** pattern for checking if the string might be a date time for pattern 4. */

	private final Pattern _formatPatternTime4 = Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z");

	/**

	 * create local instance.

	 */

	public ValueFormatHelper() {

		_formatDate.setLenient(false);

		_formatDateTimePatternDefault.setLenient(false);

		_formatDateTimePattern2.setLenient(false);

		_formatDateTimePattern3.setLenient(false);

		_formatDateTimePattern3.setTimeZone(TimeZone.getTimeZone("UTC"));

		_formatDateTimePattern4.setLenient(false);

		_formatDateTimePattern4.setTimeZone(TimeZone.getTimeZone("UTC"));

	}

	/**

	 * @return the default format for datetime values.

	 */

	public static SimpleDateFormat getDefaultDateTimeFormat() {

		final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ");

		sdf.setLenient(false);

		return sdf;

	}

	/**

	 * format value as Date string.

	 * 

	 * @param value

	 *            a date value

	 * @return formatted date.

	 */

	public String formatDate(final Date value) {

		synchronized (_formatDate) {

			return _formatDate.format(value);

		}

	}

	/**

	 * format value as DateTime string.

	 * 

	 * @param value

	 *            a datetime value

	 * @return formatted datetime string

	 */

	public String formatDateTime(final Date value) {

		synchronized (_formatDateTimePatternDefault) {

			return _formatDateTimePatternDefault.format(value);

		}

	}

	/**

	 * parse a date string.

	 * 

	 * @param dateString

	 *            a date string

	 * @return parsed Date

	 * @throws ParseException

	 *             string has wrong format

	 */

	public Date parseDate(final String dateString) throws ParseException {

		if (dateString.length() == DATE_LENGTH && _formatPatternDateDefault.matcher(dateString).matches()) {

			synchronized (_formatDate) {

				return _formatDate.parse(dateString);

			}

		} else {

			throw new ParseException("Length of date string '" + dateString + "' exceeds maximum date length of " + DATE_LENGTH, DATE_LENGTH);

		}

	}

	/**

	 * parse a date string.

	 * 

	 * @param dateString

	 *            a date string

	 * @return parsed Date if correct format, else null.

	 */

	public Date tryParseDate(final String dateString) {

		if (dateString.length() == DATE_LENGTH && _formatPatternDateDefault.matcher(dateString).matches()) {

			synchronized (_formatDate) {

				try {

					return _formatDate.parse(dateString);

				} catch (final ParseException ex) {

					; // not a date ... fine.

				}

			}

		}

		return null;

	}

	/**

	 * parse datetime string.

	 * 

	 * @param dateTimeString

	 *            a datetime string

	 * @return parsed Date

	 * @throws ParseException

	 *             string has wrong format

	 */

	public Date parseDateTime(final String dateTimeString) throws ParseException {

		final Date result;

		final int dateLen = dateTimeString.length();

		if (dateLen == DATE_TIME_LENGTH_PATTERN_DEFAULT && _formatPatternTimeDefault.matcher(dateTimeString).matches()) {

			synchronized (_formatDateTimePatternDefault) {

				result = _formatDateTimePatternDefault.parse(dateTimeString);

			}

		} else if (dateLen == DATE_TIME_LENGTH_PATTERN_2_AND_3 && _formatPatternTime2.matcher(dateTimeString).matches()) {

			synchronized (_formatDateTimePattern2) {

				result = _formatDateTimePattern2.parse(dateTimeString);

			}

		} else if (dateLen == DATE_TIME_LENGTH_PATTERN_2_AND_3 && _formatPatternTime3.matcher(dateTimeString).matches()) {

			synchronized (_formatDateTimePattern3) {

				result = _formatDateTimePattern3.parse(dateTimeString);

			}

		} else if (dateLen == DATE_TIME_LENGTH_PATTERN_4 && _formatPatternTime4.matcher(dateTimeString).matches()) {

			synchronized (_formatDateTimePattern4) {

				result = _formatDateTimePattern4.parse(dateTimeString);

			}

		} else {

			throw new ParseException("Length '" + dateTimeString.length() + "' of datetime string '" + dateTimeString

					+ "' doesn't match expected pattern length", dateTimeString.length());

		}

		return result;

	}

	/**

	 * parse datetime string.

	 * 

	 * @param dateTimeString

	 *            a datetime string

	 * @return parsed Date if correct format, else null;

	 */

	public Date tryParseDateTime(final String dateTimeString) {

		Date result = null;

		try {

			switch (dateTimeString.length()) {

			case DATE_TIME_LENGTH_PATTERN_DEFAULT:

				if (_formatPatternTimeDefault.matcher(dateTimeString).matches()) {

					synchronized (_formatDateTimePatternDefault) {

						result = _formatDateTimePatternDefault.parse(dateTimeString);

					}

				}

				break;

			case DATE_TIME_LENGTH_PATTERN_2_AND_3:

				if (_formatPatternTime2.matcher(dateTimeString).matches()) {

					synchronized (_formatDateTimePattern2) {

						result = _formatDateTimePattern2.parse(dateTimeString);

					}

				} else if (_formatPatternTime3.matcher(dateTimeString).matches()) {

					synchronized (_formatDateTimePattern3) {

						result = _formatDateTimePattern3.parse(dateTimeString);

					}

				}

				break;

			case DATE_TIME_LENGTH_PATTERN_4:

				if (_formatPatternTime4.matcher(dateTimeString).matches()) {

					synchronized (_formatDateTimePattern4) {

						result = _formatDateTimePattern4.parse(dateTimeString);

					}

				}

				break;

			default:

				break;

			}

		} catch (final ParseException ex) {

			; // not a datetime ... fine.

		}

		return result;

	}

}

package eu.dnetlib.functionality.index.query;

import java.io.IOException;

import java.util.*;

import java.util.stream.Collectors;

import java.util.stream.Stream;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.z3950.zing.cql.*;

/**

 * Use this class to cleanup a CQL tree and obtain all the options

 * 

 * @author marko & claudio

 * 

 */

public class Pruner {

	private static final Log log = LogFactory.getLog(Pruner.class); // NOPMD by marko on 11/24/08 5:02 PM

	/**

	 * All options have to be in this namespace.

	 */

	public static final String DNET_URI = "NAMESPACE";

	private String optionUri = DNET_URI;

	/**

	 * Helper method, parse a given CQL string.

	 * 

	 * @param cqlQuery

	 * @return the parsed CQLNode

	 * @throws CQLParseException

	 * @throws IOException

	 */

	CQLNode parse(final String cqlQuery) throws CQLParseException, IOException {

		return new CQLParser().parse(cqlQuery);

	}

	class Result {

		private CQLNode node;

		private List<String> options;

		public Result(final CQLNode node, final List<String> options) {

			super();

			this.node = node;

			this.options = options;

		}

		public Result(final CQLNode node, final Iterable<String> concat) {

			this.node = node;

			this.options = Lists.newArrayList(concat);

		}

		public CQLNode getNode() {

			return node;

		}

		public void setNode(final CQLNode node) {

			this.node = node;

		}

		public List<String> getOptions() {

			return options;

		}

		public void setOptions(final List<String> options) {

			this.options = options;

		}

		public Map<String, List<String>> getOptionMap() {

			Map<String, List<String>> res = new HashMap<String, List<String>>();

			for (String opt : options) {

				String[] k = opt.split("=");

				List<String> l = res.get(k[0]);

				if(l == null)

					l = new ArrayList<String>();

				l.add(k[1]);

				res.put(k[0], l);

			}

			return res;

		}

	}

	/**

	 * Remove all options from a given CQL AST and return all the options.

	 * 

	 * The CQL tree is modified.

	 * 

	 * @param root

	 *            cql tree

	 * @return pair containing a new root node and a list of options

	 */

	public Result prune(final CQLNode root) {

		return prune(new HashMap<String, String>(), root);

	}

	/**

	 * Actual recursive implementation, dispatches the implementation to the appropriate overloaded method.

	 * 

	 * @param prefixes

	 * @param root

	 * @return the pruned result

	 */

	public Result prune(final Map<String, String> prefixes, final CQLNode root) {

		if (root instanceof CQLBooleanNode)

			return prune(prefixes, (CQLBooleanNode) root);

		if (root instanceof CQLPrefixNode)

			return prune(prefixes, (CQLPrefixNode) root);

		if (root instanceof CQLSortNode)

			return prune(prefixes, (CQLSortNode) root);		

		return new Result(root, new ArrayList<String>());

	}

	/**

	 * If the current node is a cql "sort" node, just return the inner subtree.

	 * 

	 * @param prefixes

	 * @param node

	 * @return the pruned result

	 */

	public Result prune(final Map<String, String> prefixes, final CQLSortNode node) {	

		Result res = prune(prefixes, node.subtree);

		node.subtree = res.getNode();

		res.setNode(node);

		return res;

	}

	/**

	 * If the current node is a cql "prefix" node, add his namespace declaration to the current list of namespaces and

	 * return the pruned inner subtree.

	 * 

	 * If the prefix node contains only one single option element, we have to return null. (TODO: perhaps there is a

	 * better solution).

	 * 

	 * @param prefixes

	 * @param node

	 * @return the pruned result

	 */

	public Result prune(final Map<String, String> prefixes, final CQLPrefixNode node) {

		final HashMap<String, String> subPrefixes = Maps.newHashMap(prefixes);

		subPrefixes.put(node.prefix.name, node.prefix.identifier);

		if (isOption(subPrefixes, node.subtree))

			return new Result(null, Lists.newArrayList(getOption(node.subtree)));

		boolean pruneThisPrefix = node.prefix.identifier.equals(optionUri);

		if(pruneThisPrefix)

			return prune(subPrefixes, node.subtree);	

		Result res = prune(subPrefixes, node.subtree);

		node.subtree = res.getNode();

		res.setNode(node);

		return res;

	}

	/**

	 * boolean prunes are handled in the prune(prefix, node, left, right).

	 * 

	 * @param prefixes

	 * @param node

	 * @return the pruned result

	 */

	public Result prune(final Map<String, String> prefixes, final CQLBooleanNode node) {

		return prune(prefixes, node, node.left, node.right);

	}

	/**

	 * Detects if a left or right side of a boolean node is a option term, and returns the other side (recursively

	 * pruned). It also returns the accumulated options along the way.

	 * 

	 * @param prefixes

	 * @param bool

	 * @param left

	 * @param right

	 * @return the pruned result

	 */

	public Result prune(final Map<String, String> prefixes, final CQLBooleanNode bool, final CQLNode left, final CQLNode right) {

		if (isOption(prefixes, left) && isOption(prefixes, right)) {

			return new Result(null, Stream.of(trimOption(prefixes, left, right), trimOption(prefixes, right, left))

					.filter(Objects::nonNull)

					.map(i -> i.getOptions())

					.map(i -> i.stream())

					.flatMap(i -> i)

					.collect(Collectors.toList()));

		}

		Result res = anyNotNull(trimOption(prefixes, left, right), trimOption(prefixes, right, left));

		if (res != null)

			return res;

		final Result leftResult = prune(prefixes, left);

		final Result rightResult = prune(prefixes, right);

		bool.left = leftResult.getNode();

		bool.right = rightResult.getNode();

		return new Result(clean(bool), Iterables.concat(leftResult.getOptions(), rightResult.getOptions()));

	}

	public <T> T anyNotNull(T a, T b) {

		if (a != null)

			return a;

		return b;

	}

	/**

	 * Trims an option from a boolean node if one if it's sides is an option term.

	 * 

	 * Intended to be used once for each sides and then swap.

	 * 

	 * @param prefixes

	 * @param a

	 * @param b

	 * @return the pruned result

	 */

	public Result trimOption(final Map<String, String> prefixes, final CQLNode a, final CQLNode b) {

		log.debug("trim option?" + prefixes + " a " + a.toCQL());

		if (isOption(prefixes, a)) {

			log.debug("IS OPTION...");

			return trimOption(prefixes, prefixFromOption(a), getOption(a), b);

		}

		log.debug("IS NOT OPTION");

		return null;

	}

	/**

	 * prune(prefixes, bool, left, right) uses this helper method to do the dirty job:

	 * 

	 * we have to detect if a term node is a term option node. by checking the namespace uri associated with the term

	 * prefix according the the current namespace prefix scope (held in prefixes, which is passed down recursively by

	 * copy).

	 * 

	 * @param prefixes

	 * @param ns

	 * @param o

	 * @param subtree

	 * @return the pruned result

	 */

	public Result trimOption(final Map<String, String> prefixes, final String ns, final String o, final CQLNode subtree) {

		log.debug("trimming " + prefixes + " ns " + ns + " o " + o);

		final String namespaceUri = prefixes.get(ns);

		if (!optionUri.equals(namespaceUri)) {

			return null;

		}

		final Result res = prune(prefixes, subtree);

		return new Result(res.getNode(), Iterables.concat(Lists.newArrayList(o), res.getOptions()));

	}

	/**

	 * Drop a boolean node (and, or etc) if one of the sides has been dropped.

	 * 

	 * @param bool

	 * @return the pruned result

	 */

	private CQLNode clean(final CQLBooleanNode bool) {

		if (bool.left == null)

			return bool.right;

		if (bool.right == null)

			return bool.left;

		return bool;

	}

	////////////////// helpers

	public String getOption(final CQLNode node) {

		return indexFromOption(node) + "=" + termFromOption(node);

	}

	private String indexFromOption(final CQLNode node) {

		return ((CQLTermNode) node).getIndex().replaceAll("[a-z]*\\.(.+)", "$1");

	}

	private String termFromOption(final CQLNode node) {

		return ((CQLTermNode) node).getTerm();

	}

	public String prefixFromOption(final String option) {

		return option.replaceAll("([a-z]*)\\..+", "$1");

	}

	public String prefixFromOption(final CQLNode node) {

		if (node instanceof CQLTermNode)

			return prefixFromOption(((CQLTermNode) node).getIndex());

		return null;

	}

	public boolean isOption(final Map<String, String> prefixes, final String option) {

		return prefixes.containsKey(prefixFromOption(option)) && prefixes.get(prefixFromOption(option)).equals(getOptionUri());

	}

	public boolean isOption(final Map<String, String> prefixes, final CQLNode node) {

		if (node instanceof CQLTermNode)

			return isOption(prefixes, ((CQLTermNode) node).getIndex());

		return false;

	}

	public String getOptionUri() {

		return optionUri;

	}

	public void setOptionUri(String optionUri) {

		this.optionUri = optionUri;

	}

}

package eu.dnetlib.functionality.index.model;

import java.util.Date;

/**

 * Interface for a value object, used if the any object is of type string, double, boolean, long, date or date time.

 */

public interface Value extends Any {

	/**

	 * @return The string representation

	 */

	String asString();

	/**

	 * @return The double representation, an InvalidValueTypeException is thrown if the value is not a number. If value

	 *         is a string then this is tried to be converted to a Double.

	 */

	Double asDouble();

	/**

	 * @return The long representation, an InvalidValueTypeException is thrown if the value is not a number. If value is

	 *         a string then this is tried to be converted to a Long.

	 */

	Long asLong();

	/**

	 * @return The boolean representation, an InvalidValueTypeException is thrown if the value is not of type boolean.

	 *         If value is a string then this is tried to be converted to a boolean.

	 */

	Boolean asBoolean();

	/**

	 * @return The date representation, an InvalidValueTypeException is thrown if the value is not of type date or

	 *         datetime. If value is a string then this is tried to be converted to a date.

	 */

	Date asDate();

	/**

	 * @return The date time representation, an InvalidValueTypeException is thrown if the value is not of type

	 *         datetime. If value is a string then this is tried to be converted to a datetime.

	 */

	Date asDateTime();

	/**

	 * @return the value object

	 */

	Object getObject();

}

package eu.dnetlib.functionality.index.query;

/**

 * The Class IndexQuery.

 */

public interface IndexQuery {

	IndexQuery setQueryOffset(int offset);

	IndexQuery setQueryLimit(int limit);

}

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

    <parent>

        <groupId>eu.dnetlib</groupId>

        <artifactId>dnet45-parent</artifactId>

        <version>1.0.0</version>

        <relativePath />

    </parent>

    <modelVersion>4.0.0</modelVersion>

    <groupId>eu.dnetlib</groupId>

    <artifactId>dnet-index-client</artifactId>

    <packaging>jar</packaging>

    <version>3.0.0</version>

    <scm>

        <developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-index-client/tags/dnet-index-client-3.0.0</developerConnection>

    </scm>

    <dependencies>

        <dependency>

            <groupId>org.springframework</groupId>

            <artifactId>spring-beans</artifactId>

            <version>${spring.version}</version>

        </dependency>

        <dependency>

            <groupId>junit</groupId>

            <artifactId>junit</artifactId>

            <version>${junit.version}</version>

            <scope>test</scope>

        </dependency>

        <dependency>

            <groupId>org.mockito</groupId>

            <artifactId>mockito-core</artifactId>

            <version>${mockito.version}</version>

            <scope>test</scope>

        </dependency>

        <dependency>

            <groupId>com.google.code.gson</groupId>

            <artifactId>gson</artifactId>

            <version>${google.gson.version}</version>

        </dependency>

        <dependency>

            <groupId>com.mycila</groupId>

            <artifactId>xmltool</artifactId>

            <version>3.3</version>

        </dependency>

        <dependency>

            <groupId>org.apache.solr</groupId>

            <artifactId>solr-solrj</artifactId>

            <version>7.5.0</version>

            <exclusions>

                <exclusion>

                    <artifactId>wstx-asl</artifactId>

                    <groupId>org.codehaus.woodstox</groupId>

                </exclusion>

                <exclusion>

                    <artifactId>jcl-over-slf4j</artifactId>

                    <groupId>org.slf4j</groupId>

                </exclusion>

            </exclusions>

        </dependency>

        <dependency>

            <groupId>eu.dnetlib</groupId>

            <artifactId>dnet-index-solr-common</artifactId>

            <version>[3.0.0,4.0.0)</version>

        </dependency>

        <dependency>

            <groupId>eu.dnetlib</groupId>

            <artifactId>cnr-service-common</artifactId>

            <version>[2.1.0,3.0.0)</version>

        </dependency>

        <dependency>

            <groupId>eu.dnetlib</groupId>

            <artifactId>cnr-cql-utils</artifactId>

            <version>[2.0.0,3.0.0)</version>

        </dependency>

        <dependency>

            <groupId>eu.dnetlib</groupId>

            <artifactId>dnet-data-provision-rmi</artifactId>

            <version>[1.0.0,2.0.0)</version>

        </dependency>

    </dependencies>

</project>

package eu.dnetlib.functionality.index.query;

public enum QueryLanguage {

	CQL, SOLR

}

package eu.dnetlib.functionality.index.query;

import java.util.HashMap;

import java.util.Map;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import eu.dnetlib.functionality.index.client.IndexClientException;

import eu.dnetlib.functionality.index.utils.MDFormatReader;

import eu.dnetlib.functionality.index.utils.MetadataReference;

import eu.dnetlib.functionality.index.utils.ServiceTools;

public class Weights extends HashMap<MetadataReference, Map<String, String>> {

	private static final long serialVersionUID = -3517914310574484765L;

	private static final Log log = LogFactory.getLog(Weights.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Autowired

	private ServiceTools serviceTools;

	@Autowired

	private MDFormatReader mdFormatReader;

	public Weights() {

		super();

	}

	public void initialize() throws IndexClientException {

		log.info("initializing weights");

		for (MetadataReference mdRef : serviceTools.listMDRefs()) {

			put(mdRef, mdFormatReader.getAttributeMap(mdRef, "weight"));

		}

		log.info("weights initialization completed");

	}

	@Override

	public Map<String, String> put(final MetadataReference mdRef, final Map<String, String> w) {

		log.info("[" + mdRef + "]" + " adding weights: " + w);

		return super.put(mdRef, w);

	}

	@Override

	public Map<String, String> get(final Object mdRef) {

		Map<String, String> map = super.get(mdRef);

		return map != null ? map : initAndGet(mdRef);

	}

	private Map<String, String> initAndGet(final Object mdRef) {

		try {

			initialize();

		} catch (IndexClientException e) {

			throw new RuntimeException(e);

		}

		return super.get(mdRef);

	}

}

package eu.dnetlib.functionality.index.query;

import java.util.List;

import com.google.common.collect.BiMap;

import com.google.common.collect.HashBiMap;

import com.google.common.collect.Lists;

import eu.dnetlib.functionality.index.client.IndexClientException;

import org.junit.Assert;

import org.junit.Before;

import org.junit.Test;

public class SolrIndexQueryFactoryTest {

	IndexQueryFactory factory;

	BiMap<String, String> browsingAliases = HashBiMap.create();

	@Before

	public void setUp() throws Exception {

		factory = new SolrIndexQueryFactory();

		browsingAliases.put("field1", "field1ForBrowsing");

		browsingAliases.put("field2", "field2ForBrowsing");

	}

	@Test

	public void testGetBrowsableFields() throws IndexClientException {

		List<String> browsables = factory.getBrowsableFields(Lists.newArrayList("field1", "field3", "field2"), browsingAliases);

		Assert.assertEquals(Lists.newArrayList("field1ForBrowsing", "field3", "field2ForBrowsing"), browsables);

	}

}

package eu.dnetlib.functionality.index.utils;

public class MetadataReferenceFactory {

	public static MetadataReference getMetadata(final String format, final String layout, final String interpretation) {

		return new MetadataReference(format, layout, interpretation);

	}

	public static MetadataReference decodeMetadata(final String encoded) {

		String[] split = encoded.split("-");

		if (split.length == 3) return getMetadata(split[0], split[1], split[2]);

		throw new IllegalStateException("malformed metadata reference: " + encoded);

	}

}

package eu.dnetlib.functionality.index.utils;

import com.google.common.collect.ComparisonChain;

/**

 * Container class for metadata format, layout and interpretation.

 * 

 * @author claudio

 *

 */

public class MetadataReference {

	/**

	 * Metadata format.

	 */

	private String format;

	/**

	 * Metadata layout.

	 */

	private String layout;

	/**

	 * Metadata interpretation.

	 */

	private String interpretation;

	/**

	 * Constructor for MetadataReference.

	 * 

	 * @param format

	 * 			Metadata format

	 * @param layout

	 * 			Metadata layout

	 * @param interpretation

	 * 			Metadata interpretation

	 */

	public MetadataReference(final String format, final String layout, final String interpretation) {

		this.format = format;

		this.layout = layout;

		this.interpretation = interpretation;

	}

	public String getFormat() {

		return format;

	}

	public String getLayout() {

		return layout;

	}

	public String getInterpretation() {

		return interpretation;

	}

	@Override

	public boolean equals(final Object that) {

		if (!(that instanceof MetadataReference))

			return false;

		final MetadataReference mdRef = (MetadataReference) that;

		return ComparisonChain.start()

	        .compare(this.format, mdRef.getFormat())

	        .compare(this.layout, mdRef.getLayout())

	        .compare(this.interpretation, mdRef.getInterpretation())

	        .result() == 0;		

	}

	@Override

	public int hashCode() {

		return getFormat().hashCode() + getLayout().hashCode() + getInterpretation().hashCode();

	}	

	@Override

	public String toString() {

		return getFormat() + "-" + getLayout() + "-" + getInterpretation();

	}

}

package eu.dnetlib.functionality.index.query;

import eu.dnetlib.functionality.index.client.IndexClientException;

import eu.dnetlib.functionality.index.utils.MetadataReference;

import org.apache.solr.client.solrj.response.QueryResponse;

/**

 * The Class SolrIndexQueryResponseFactory.

 */

public class SolrIndexQueryResponseFactory extends QueryResponseFactory<QueryResponse> {

	/**

	 * {@inheritDoc}

	 * 

	 * @throws IndexClientException

	 *

	 * @see QueryResponseFactory#getQueryResponseParser(IndexQueryResponse,

	 *      MetadataReference)

	 */

	@Override

	public QueryResponseParser getQueryResponseParser(final IndexQueryResponse<QueryResponse> queryRsp, final MetadataReference mdRef)

			throws IndexClientException {

		QueryResponse response = queryRsp.getContextualQueryResponse();

		return new SolrResponseParser(highlightUtils, browseAliases.get(mdRef), returnEmptyFields, includeRanking, response);

	}

}

package eu.dnetlib.functionality.index.client.solr;

import java.io.IOException;

import java.text.SimpleDateFormat;

import java.util.*;

import com.google.common.collect.BiMap;

import com.google.common.collect.Maps;

import eu.dnetlib.data.provision.index.rmi.BrowsingRow;

import eu.dnetlib.data.provision.index.rmi.GroupResult;

import eu.dnetlib.data.provision.index.rmi.IndexServiceException;

import eu.dnetlib.functionality.cql.CqlValueTransformerMap;

import eu.dnetlib.functionality.index.client.IndexClient;

import eu.dnetlib.functionality.index.client.IndexClientException;

import eu.dnetlib.functionality.index.client.response.BrowseEntry;

import eu.dnetlib.functionality.index.client.response.BrowseValueEntry;

import eu.dnetlib.functionality.index.client.response.LookupResponse;

import eu.dnetlib.functionality.index.model.Any.ValueType;

import eu.dnetlib.functionality.index.query.*;

import eu.dnetlib.functionality.index.solr.cql.SolrTypeBasedCqlValueTransformerMapFactory;

import eu.dnetlib.functionality.index.solr.feed.StreamingInputDocumentFactory;

import eu.dnetlib.functionality.index.utils.MetadataReference;

import eu.dnetlib.functionality.index.utils.ZkServers;

import eu.dnetlib.miscutils.datetime.HumanTime;

import eu.dnetlib.miscutils.functional.UnaryFunction;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.SolrClient;

import org.apache.solr.client.solrj.SolrQuery;

import org.apache.solr.client.solrj.SolrServerException;

import org.apache.solr.client.solrj.impl.CloudSolrClient;

import org.apache.solr.client.solrj.request.LukeRequest;

import org.apache.solr.client.solrj.response.LukeResponse;

import org.apache.solr.client.solrj.response.LukeResponse.FieldInfo;

import org.apache.solr.client.solrj.response.QueryResponse;

import org.apache.solr.client.solrj.response.UpdateResponse;

import org.apache.solr.common.SolrInputDocument;

/**

 * The Class SolrIndexClient.

 */

public class SolrIndexClient implements IndexClient {

	private static final Log log = LogFactory.getLog(SolrIndexClient.class);

	private static final String INDEX_RECORD_RESULT_FIELD = "dnetResult";

	private static String ZK_ADDRESS = "address";

	/** The format. */

	private String format;

	/** The layout. */

	private String layout;

	/** The interpretation. */

	private String interpretation;

	protected Map<String, String> serviceProperties;

	/** The client. */

	private CloudSolrClient client;

	private SolrIndexQueryFactory solrIndexQueryFactory;

	/** The query response factory. */

	private QueryResponseFactory<QueryResponse> queryResponseFactory;

	private SolrTypeBasedCqlValueTransformerMapFactory tMapFactory;

	/**

	 * The Constructor.

	 *

	 * @param format

	 *            the format

	 * @param layout

	 *            the layout

	 * @param interpretation

	 *            the interpretation

	 * @param serviceProperties

	 *            the service properties

	 * @param tMapFactory

	 */

	public SolrIndexClient(final String format, final String layout, final String interpretation, final Map<String, String> serviceProperties,

			final SolrIndexQueryFactory indexQueryFactory, final QueryResponseFactory<QueryResponse> queryResponseFactory,

			final SolrTypeBasedCqlValueTransformerMapFactory tMapFactory) {

		this.format = format;

		this.layout = layout;

		this.interpretation = interpretation;

		this.serviceProperties = serviceProperties;

		this.solrIndexQueryFactory = indexQueryFactory;

		this.queryResponseFactory = queryResponseFactory;

		this.tMapFactory = tMapFactory;

		log.debug(String.format("Created a new instance of the index of type %s-%s-%s", format, layout, interpretation));

	}

	/**

	 * Do delete.

	 *

	 * @param query

	 *            the CQL query

	 * @return true, if do delete

	 * @throws IndexServiceException

	 *             the index service exception

	 */

	@Override

	public long delete(final String query) throws IndexClientException {

		try {

			log.debug("delete by query: " + query);

			MetadataReference mdRef = new MetadataReference(getFormat(), getLayout(), getInterpretation());

			SolrIndexQuery translatedQuery = (SolrIndexQuery) solrIndexQueryFactory.getIndexQuery(QueryLanguage.CQL, query, this, mdRef);

			String tquery = translatedQuery.getQuery();

			translatedQuery.setQueryLimit(0);

			SolrIndexQueryResponse rsp = new SolrIndexQueryResponse(getClient().query(translatedQuery));

			QueryResponseParser responseParser = queryResponseFactory.getQueryResponseParser(rsp, mdRef);

			long total = responseParser.getNumFound();

			getClient().deleteByQuery(tquery);

			getClient().commit();

			return total;

		} catch (Exception e) {

			throw new IndexClientException("unable to run delete by query: " + query, e);

		}

	}

	/**

	 * {@inheritDoc}

	 *