D-Net

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-index-solr-client/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "dnet-index-solr-client"}

package eu.dnetlib.functionality.index.query;

import java.util.List;

import org.junit.Assert;

import org.junit.Before;

import org.junit.Test;

import com.google.common.collect.BiMap;

import com.google.common.collect.HashBiMap;

import com.google.common.collect.Lists;

import eu.dnetlib.functionality.index.client.IndexClientException;

public class SolrIndexQueryFactoryTest {

	IndexQueryFactory factory;

	BiMap<String, String> browsingAliases = HashBiMap.create();

	@Before

	public void setUp() throws Exception {

		factory = new SolrIndexQueryFactory();

		browsingAliases.put("field1", "field1ForBrowsing");

		browsingAliases.put("field2", "field2ForBrowsing");

	}

	@Test

	public void testGetBrowsableFields() throws IndexClientException {

		List<String> browsables = factory.getBrowsableFields(Lists.newArrayList("field1", "field3", "field2"), browsingAliases);

		Assert.assertEquals(Lists.newArrayList("field1ForBrowsing", "field3", "field2ForBrowsing"), browsables);

	}

}

package eu.dnetlib.functionality.index.solr.cql;

import java.util.Map;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.functionality.index.model.Any.ValueType;

import eu.dnetlib.miscutils.functional.UnaryFunction;

/**

 * Factory for the SolrTypeBasedCqlValueTransformerMap class objects

 * 

 * @author claudio

 * 

 */

public class SolrTypeBasedCqlValueTransformerMapFactory {

	/**

	 * Map of functions, injected via spring.

	 */

	private Map<String, UnaryFunction<String, String>> transformerMap;

	/**

	 * Method returns a new instance of SolrTypeBasedCqlValueTransformerMap.

	 * 

	 * @param schema

	 * @return

	 */

	public SolrTypeBasedCqlValueTransformerMap getIt(final Map<String, ValueType> schema) {

		return new SolrTypeBasedCqlValueTransformerMap(schema, getTransformerMap());

	}

	@Required

	public void setTransformerMap(Map<String, UnaryFunction<String, String>> transformerMap) {

		this.transformerMap = transformerMap;

	}

	public Map<String, UnaryFunction<String, String>> getTransformerMap() {

		return transformerMap;

	}

}

package eu.dnetlib.functionality.index.solr.cql;

import eu.dnetlib.miscutils.functional.UnaryFunction;

/**

 * Simply and not very roboust normalizer for solr dates. Basically it handles well yyyy-mm-dd and

 * yyyy-mm-ddThh:mm:ssZ

 * 

 * @author marko

 * 

 */

public class SimpleDateValueTransformer implements UnaryFunction<String, String> {

	@Override

	public String evaluate(final String value) {

		if (!value.endsWith("Z"))

			return value + "T00:00:00Z";

		return value;

	}

}

package eu.dnetlib.functionality.index.solr.cql;

import java.util.Map;

import eu.dnetlib.functionality.cql.CqlValueTransformerMap;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.common.SolrException;

import eu.dnetlib.functionality.index.model.Any.ValueType;

import eu.dnetlib.miscutils.functional.IdentityFunction;

import eu.dnetlib.miscutils.functional.UnaryFunction;

/**

 * This class maps the fields in the given index schema with a transformation rule.

 * 

 * @author marko

 * 

 */

public class SolrTypeBasedCqlValueTransformerMap implements CqlValueTransformerMap {

	/**

	 * logger.

	 */

	private static final Log log = LogFactory.getLog(SolrTypeBasedCqlValueTransformerMap.class); // NOPMD by marko on 11/24/08 5:02 PM

	/**

	 * Index schema.

	 */

	private final Map<String, ValueType> schema;

	/**

	 * Map of functions.

	 */

	private final Map<String, UnaryFunction<String, String>> transformerMap;

	/**

	 * Create value transformer map bound to a specific schema

	 * @param schema

	 * @param transformerMap

	 */

	public SolrTypeBasedCqlValueTransformerMap(final Map<String, ValueType> schema, final Map<String, UnaryFunction<String, String>> transformerMap) {

		this.schema = schema;

		this.transformerMap = transformerMap;

	}

	/**

	 * {@inheritDoc}

	 * 

	 * @see eu.dnetlib.functionality.cql.CqlValueTransformerMap#transformerFor(java.lang.String)

	 */

	@Override

	public UnaryFunction<String, String> transformerFor(final String fieldName) {

		try {

			final ValueType field = schema.get(fieldName);

			if (field != null) {

				UnaryFunction<String, String> res = transformerMap.get(field.name());

				if (res != null) {

					return res;

				}

			}

		} catch (SolrException e) {

			log.debug("cannot find field", e);

		}

		return new IdentityFunction<String>();

	}

}

package eu.dnetlib.functionality.index.solr.utils;

import org.apache.oro.text.perl.Perl5Util;

import eu.dnetlib.miscutils.functional.UnaryFunction;

public class HighlightUtils implements UnaryFunction<String, String> {

	public final static String DEFAULT_HL_PRE = "[hl]";

	public final static String DEFAULT_HL_POST = "[/hl]";

	private static String CLEAN_HEADER = "s#\\[/?hl\\]##gm";

	private static String CLEAN_REGEX_OPEN = "<([^>]*)\\[hl\\]([^>]*)>";

	private static String CLEAN_REGEX_CLOSE = "<([^>]*)\\[\\/hl\\]([^>]*)>";

	// private static String CLEAN_REGEX_OPEN = "s#<([^>]*)\\[hl\\]([^>]*)>#<$1$2>#gm";

	// private static String CLEAN_REGEX_CLOSE = "s#<([^>]*)\\[\\/hl\\]([^>]*)>#<$1$2>#gm";

	private Perl5Util p5util = new Perl5Util();

	@Override

	public String evaluate(final String doc) {

		String[] chunk = doc.split("</header>");

		String string = cleanHeader(chunk[0]) + "</header>" + cleanBody(chunk[1]);

		return string;

	}

	private String cleanHeader(final String header) {

		return p5util.substitute(CLEAN_HEADER, header);

	}

	// TODO: implement a faster way to do this

	private String cleanBody(final String body) {

		String res = body.replaceAll(CLEAN_REGEX_OPEN, "<$1$2>").replaceAll(CLEAN_REGEX_CLOSE, "<$1$2>");

		if (res.equals(body)) return res;

		return cleanBody(res);

	}

}

package eu.dnetlib.functionality.index.model.util;

import java.util.Iterator;

import java.util.Map.Entry;

import org.apache.solr.common.SolrDocumentList;

import org.apache.solr.common.util.NamedList;

import eu.dnetlib.functionality.index.model.AnyMap;

import eu.dnetlib.functionality.index.model.DataFactory;

import eu.dnetlib.functionality.index.model.InvalidValueTypeException;

import eu.dnetlib.functionality.index.model.Value;

import eu.dnetlib.functionality.index.model.impl.DefaultDataFactoryImpl;

/**

 * The Class AnySolrUtil.

 */

public class AnySolrUtil extends AnyUtil {

	/**

	 * Convert named list to any map.

	 * 

	 * @param list

	 *            the list

	 * @param map

	 *            the map

	 * @return the any map

	 */

	@SuppressWarnings("unchecked")

	public static AnyMap convertNamedListToAnyMap(final NamedList<Object> list, final AnyMap map) {

		final Iterator<Entry<String, Object>> it = list.iterator();

		while (it.hasNext()) {

			Entry<String, Object> entry = it.next();

			final String key = entry.getKey();

			final Object obj = entry.getValue();

			if (obj instanceof NamedList<?>) {

				final AnyMap subMap = map.getMap(key, true);

				convertNamedListToAnyMap((NamedList<Object>) obj, subMap);

			} else if (obj instanceof SolrDocumentList) {

				SolrDocumentList docList = (SolrDocumentList) obj;

				AnyMap response = DataFactory.DEFAULT.createAnyMap();

				response.put("numFound", docList.getNumFound());

				response.put("start", docList.getStart());

				response.put("maxScore", docList.getMaxScore());

				response.put("docs", objectToAny(obj));

				map.put("response", response);

			} else {

				try {

					final Value value = DataFactory.DEFAULT.autoConvertValue(obj);

					map.put(key, value);

				} catch (InvalidValueTypeException exception) {

					; // skip

				}

			}

		}

		return map;

	}

	/**

	 * Convert named list to any map.

	 * 

	 * @param list

	 *            the list

	 * @return the any map

	 */

	public static AnyMap convertNamedListToAnyMap(final NamedList<Object> list) {

		return convertNamedListToAnyMap(list, DefaultDataFactoryImpl.INSTANCE.createAnyMap());

	}

}

package eu.dnetlib.functionality.index.model.util;

import java.util.Collection;

import java.util.HashMap;

import java.util.Map;

import org.apache.solr.client.solrj.response.SpellCheckResponse.Collation;

import org.apache.solr.common.SolrInputDocument;

import org.apache.solr.common.SolrInputField;

import eu.dnetlib.functionality.index.model.Any.ValueType;

import eu.dnetlib.functionality.index.model.document.AbstractIndexDocument;

// TODO: Auto-generated Javadoc

/**

 * The Class SolrIndexDocument an implementation of the index document for SOLR.

 */

public class SolrIndexDocument extends AbstractIndexDocument {

	/**

	 * Instantiates a new solr index document.

	 *

	 * @param schema

	 *            the schema

	 * @param dsId

	 *            the ds id

	 */

	public SolrIndexDocument(final Map<String, ValueType> schema, final String dsId) {

		super(schema, dsId);

	}

	/**

	 * Instantiates a new solr index document.

	 *

	 * @param schema

	 *            the schema

	 * @param dsId

	 *            the ds id

	 * @param solrDocument

	 *            the solr document

	 */

	public SolrIndexDocument(final Map<String, ValueType> schema, final String dsId, final SolrInputDocument solrDocument) {

		super(schema, dsId);

		addFields(solrDocument);

	}

	/**

	 * Adds the fields.

	 *

	 * @param solrDocument

	 *            the solr document

	 */

	private void addFields(final SolrInputDocument solrDocument) {

		for (String name : solrDocument.getFieldNames()) {

			Collection<Object> fieldValues = solrDocument.getFieldValues(name);			

			if(fieldValues.size()>1)			

				addField(name, fieldValues);

			else if (fieldValues.size()==1)

				addField(name, fieldValues.iterator().next());

		}

	}

	/**

	 * Sets the content.

	 *

	 * @param solrDocument

	 *            the new content

	 */

	public void setContent(final SolrInputDocument solrDocument) {

		addFields(solrDocument);

	}

	/**

	 * Gets the solr document.

	 *

	 * @return the solr document

	 */

	public SolrInputDocument getSolrDocument() {

		Map<String, SolrInputField> data = new HashMap<String, SolrInputField>();

		for (String key : fields.keySet()) {

			SolrInputField solrField = new SolrInputField(key);

			for (Object o : fields.get(key)) {

				solrField.addValue(o, 1.0f);

			}

			data.put(key, solrField);

		}

		return new SolrInputDocument(data);

	}

}

package eu.dnetlib.functionality.index.query;

import org.apache.solr.client.solrj.response.QueryResponse;

import eu.dnetlib.functionality.index.client.IndexClientException;

import eu.dnetlib.functionality.index.utils.MetadataReference;

/**

 * The Class SolrIndexQueryResponseFactory.

 */

public class SolrIndexQueryResponseFactory extends QueryResponseFactory<QueryResponse> {

	/**

	 * {@inheritDoc}

	 * 

	 * @throws IndexClientException

	 *

	 * @see eu.dnetlib.functionality.index.query.QueryResponseFactory#getQueryResponseParser(eu.dnetlib.functionality.index.query.IndexQueryResponse,

	 *      eu.dnetlib.functionality.index.utils.MetadataReference)

	 */

	@Override

	public QueryResponseParser getQueryResponseParser(final IndexQueryResponse<QueryResponse> queryRsp, final MetadataReference mdRef)

			throws IndexClientException {

		QueryResponse response = queryRsp.getContextualQueryResponse();

		return new SolrResponseParser(highlightUtils, browseAliases.get(mdRef), returnEmptyFields, includeRanking, response);

	}

}

package eu.dnetlib.functionality.index.query;

import java.util.Arrays;

import eu.dnetlib.functionality.cql.lucene.TranslatedQuery;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import eu.dnetlib.functionality.index.client.AbstractIndexClient;

import eu.dnetlib.functionality.index.query.Pruner.Result;

import eu.dnetlib.functionality.index.solr.utils.HighlightUtils;

import eu.dnetlib.functionality.index.utils.IndexFieldUtility;

/**

 * A factory for creating SolrIndexQuery objects.

 */

public class SolrIndexQueryFactory extends IndexQueryFactory {

	/**

	 * logger.

	 */

	private static final Log log = LogFactory.getLog(SolrIndexQueryFactory.class);

	/** The Property name SERVICE_HIGHLIGHT_ENABLE. */

	private static final String SERVICE_HIGHLIGHT_ENABLE = "service.index.solr.highlight.enable";

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.functionality.index.query.IndexQueryFactory#newInstance(eu.dnetlib.functionality.index.cql.TranslatedQuery,

	 * eu.dnetlib.functionality.index.query.Pruner.Result, eu.dnetlib.functionality.index.query.QueryLanguage)

	 */

	@Override

	protected IndexQuery newInstance(final TranslatedQuery cql, final Result res, final QueryLanguage lang) {

		switch (lang) {

		case CQL:

			return new SolrIndexQuery(cql, res.getOptionMap());

		case SOLR:

			return new SolrIndexQuery(res.getNode().toCQL(), res.getOptionMap());

		default:

			throw new IllegalArgumentException("invalid query language: " + lang);

		}

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.functionality.index.query.IndexQueryFactory#setQueryOptions(eu.dnetlib.functionality.index.query.IndexQuery,

	 * eu.dnetlib.functionality.index.IndexServerDAO)

	 */

	@Override

	protected IndexQuery setQueryOptions(final IndexQuery indexQuery, final AbstractIndexClient client) {

		final SolrIndexQuery solrQuery = (SolrIndexQuery) indexQuery;

		boolean isHighlightEnabled = Boolean.parseBoolean(client.getServiceProperties().get(SERVICE_HIGHLIGHT_ENABLE));

		if (solrQuery.getHighlight() & isHighlightEnabled) {

			solrQuery.setHighlightFragsize(0).setHighlightSnippets(1).setHighlightSimplePre(HighlightUtils.DEFAULT_HL_PRE)

					.setHighlightSimplePost(HighlightUtils.DEFAULT_HL_POST).addHighlightField(IndexFieldUtility.RESULT)

					.addField(IndexFieldUtility.INDEX_RECORD_ID);

		}

		solrQuery.addField(IndexFieldUtility.RESULT);

		if (solrQuery.getFacetFields() != null) {

			log.debug("getFacetFields() " + Arrays.asList(solrQuery.getFacetFields()));

			solrQuery.setFacetMinCount(1);

		}

		return solrQuery;

	}

}

package eu.dnetlib.functionality.index.query;

import static eu.dnetlib.miscutils.collections.MappedCollection.listMap;

import java.util.Collection;

import java.util.List;

import java.util.Map;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.response.FacetField;

import org.apache.solr.client.solrj.response.FacetField.Count;

import org.apache.solr.client.solrj.response.QueryResponse;

import org.apache.solr.common.SolrDocument;

import org.apache.solr.common.SolrDocumentList;

import com.google.common.base.Predicate;

import com.google.common.collect.BiMap;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import eu.dnetlib.data.provision.index.rmi.BrowsingRow;

import eu.dnetlib.data.provision.index.rmi.GroupResult;

import eu.dnetlib.functionality.index.utils.IndexFieldUtility;

import eu.dnetlib.miscutils.functional.UnaryFunction;

/**

 * The Class SolrResponseParser.

 */

public class SolrResponseParser extends QueryResponseParser {

	/**

	 * logger.

	 */

	private static final Log log = LogFactory.getLog(SolrResponseParser.class);

	/**

	 * Lower level response.

	 */

	private QueryResponse queryRsp = null;

	/** The wrapper rank. */

	protected final UnaryFunction<String, SolrDocument> wrapperRank = new UnaryFunction<String, SolrDocument>() {

		@Override

		public String evaluate(final SolrDocument doc) {

			return addRanking(getSingleField(doc, IndexFieldUtility.RESULT), getSingleField(doc, IndexFieldUtility.SCORE_FIELD));

		}

	};

	/** The wrapper no rank. */

	protected final UnaryFunction<String, SolrDocument> wrapperNoRank = new UnaryFunction<String, SolrDocument>() {

		@Override

		public String evaluate(final SolrDocument doc) {

			return wrap(getSingleField(doc, IndexFieldUtility.RESULT));

		}

	};

	/**

	 * The Constructor.

	 *

	 * @param highlightUtils

	 *            the highlight utils

	 * @param aliases

	 *            the aliases

	 * @param returnEmptyFields

	 *            the return empty fields

	 * @param includeRanking

	 *            the include ranking

	 * @param response

	 *            the response

	 */

	public SolrResponseParser(final UnaryFunction<String, String> highlightUtils, final BiMap<String, String> aliases, final boolean returnEmptyFields,

			final boolean includeRanking, final QueryResponse response) {

		super(highlightUtils, aliases, returnEmptyFields, includeRanking);

		this.queryRsp = response;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.query.QueryResponseParser#getNumFound()

	 */

	@Override

	public long getNumFound() {

		return this.queryRsp.getResults().getNumFound();

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.query.QueryResponseParser#getQueryTime()

	 */

	@Override

	public int getQueryTime() {

		return queryRsp.getQTime();

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.query.QueryResponseParser#getElapsedTime()

	 */

	@Override

	public long getElapsedTime() {

		return queryRsp.getElapsedTime();

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.query.QueryResponseParser#getStatus()

	 */

	@Override

	public String getStatus() {

		return String.valueOf(queryRsp.getStatus());

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.query.QueryResponseParser#getCurrentSize()

	 */

	@Override

	public int getCurrentSize() {

		return queryRsp.getResults().size();

	}

	/**

	 * Gets the query response.

	 *

	 * @return the query response

	 */

	public QueryResponse getQueryResponse() {

		return queryRsp;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.query.QueryResponseParser#getResults()

	 */

	@Override

	public List<String> getResults() {

		return asRankedList(queryRsp.getResults());

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.query.QueryResponseParser#getNumberOfBrowsingResults()

	 */

	@Override

	public Long getNumberOfBrowsingResults() {

		List<FacetField> ffList = queryRsp.getFacetFields();

		Long maxCount = 0L;

		if (ffList != null) {

			for (FacetField ff : ffList) {

				if (ff != null) {

					Long countFacets = countFacets(ff.getValues());

					if (countFacets > maxCount) {

						maxCount = countFacets;

					}

				}

			}

		}

		return maxCount;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.query.QueryResponseParser#getBrowsingResults()

	 */

	@Override

	public List<BrowsingRow> getBrowsingResults() {

		List<BrowsingRow> bresList = Lists.newArrayList();

		List<GroupResult> facets = Lists.newArrayList();

		final List<FacetField> ffList = queryRsp.getFacetFields();

		Long numberOfBrowsingResults = getNumberOfBrowsingResults();

		for (int i = 0; (ffList != null) && (i < numberOfBrowsingResults); i++) {

			for (FacetField ff : ffList) {

				String name = null;

				if (aliases != null) {

					name = aliases.inverse().get(ff.getName());

				}

				// fix #1456

				if (name == null) {

					name = ff.getName();

				}

				final Count facet = getFacet(ff, i);

				if ((facet != null) && (facet.getCount() > 0)) {

					final String value = facet.getName();

					final int count = (int) facet.getCount();

					if (returnEmptyFields || !value.isEmpty()) {

						facets.add(new GroupResult(name, value, count));

					}

				}

			}

			if (facets.size() > 0) {

				bresList.add(new BrowsingRow(Lists.newArrayList(facets)));

				facets.clear();

			}

		}

		if (log.isDebugEnabled()) {

			log.debug("BrowsingResult size: " + bresList.size());

		}

		return bresList;

	}

	// /////////////// helpers

	/**

	 * Gets the facet.

	 *

	 * @param ff

	 *            the ff

	 * @param pos

	 *            the pos

	 * @return the facet

	 */

	private Count getFacet(final FacetField ff, final int pos) {

		if ((ff.getValues() == null) || (pos >= ff.getValues().size())) return null;

		return ff.getValues().get(pos);

	}

	/**

	 * Given SolrDocumentList, return a List of Strings, representing it.

	 *

	 * @param documentList

	 *            the document list

	 * @return the list< string>

	 */

	private List<String> asRankedList(final SolrDocumentList documentList) {

		UnaryFunction<String, SolrDocument> wrapper = includeRanking ? wrapperRank : wrapperNoRank;

		if (queryRsp.getHighlighting() != null) return listMap(listMap(documentList, new UnaryFunction<String, SolrDocument>() {

			@Override

			public String evaluate(final SolrDocument doc) {

				String score = getSingleField(doc, IndexFieldUtility.SCORE_FIELD);

				String hl = getHighlighting(getSingleField(doc, IndexFieldUtility.INDEX_RECORD_ID));

				String res = hl != null ? hl : getSingleField(doc, IndexFieldUtility.RESULT);

				return includeRanking ? addRanking(res, score) : wrap(res);

			}

		}), highlightUtils);

		return listMap(documentList, wrapper);

	}

	/**

	 * Converts a String document to

	 *

	 * <record rank="score"> [document] </record>.

	 *

	 * @param doc

	 *            the doc

	 * @param score

	 *            the score

	 * @return the string

	 */

	private String addRanking(final String doc, final String score) {

		return new String("<record rank=\"" + score + "\">" + doc + "</record>");

	}

	/**

	 * Wraps the given document as <record> [document] </record>.

	 *

	 * @param doc

	 *            the doc

	 * @return the string

	 */

	private String wrap(final String doc) {

		return new String("<record>" + doc + "</record>");

	}

	/**

	 * Gets the single field.

	 *

	 * @param doc

	 *            the doc

	 * @param fieldName

	 *            the field name

	 * @return the single field

	 */

	@SuppressWarnings("unchecked")

	protected String getSingleField(final SolrDocument doc, final String fieldName) {

		Object value = doc.getFieldValue(fieldName);

		if (value instanceof Collection) return Iterables.getOnlyElement((Iterable<String>) value);

		return String.valueOf(value);

	}

	/**

	 * Gets the highlighting.

	 *

	 * @param docId

	 *            the doc id

	 * @return the highlighting

	 */

	private String getHighlighting(final String docId) {

		final Map<String, List<String>> highlight = queryRsp.getHighlighting().get(docId);

		String result = new String();

		if ((highlight != null) && (highlight.get(IndexFieldUtility.RESULT) != null)) {

			for (String s : highlight.get(IndexFieldUtility.RESULT)) {

				result = result.concat(s);

			}

			return result;

		}

		return null;

	}

	/**

	 * helper method.

	 *

	 * @param facets

	 *            the list of facets to analyze

	 * @return the number of non-empty facets in the list whose count is greater than zero

	 */

	private Long countFacets(final List<Count> facets) {

		if (facets == null) return 0L;

		return (long) Iterables.size(Iterables.filter(facets, new Predicate<Count>() {

			@Override

			public boolean apply(final Count c) {

				return (c != null) && (c.getName() != null) && !c.getName().isEmpty() && (c.getCount() > 0);

			}

		}));

	}

	@Override

	public long getStart() {

		// TODO Auto-generated method stub

		return queryRsp.getResults().getStart();

	}

}

package eu.dnetlib.functionality.index.query;

import org.apache.solr.client.solrj.response.QueryResponse;

/**

 * The Class SolrIndexQueryResponse.

 */

public class SolrIndexQueryResponse implements IndexQueryResponse<QueryResponse> {

	private QueryResponse solrQueryResponse;

	public SolrIndexQueryResponse(final QueryResponse solrQueryResponse) {

		this.solrQueryResponse = solrQueryResponse;

	}

	@Override

	public QueryResponse getContextualQueryResponse() {

		return solrQueryResponse;

	}

}

package eu.dnetlib.functionality.index.query;

import java.util.List;

import java.util.Map;

import eu.dnetlib.functionality.cql.lucene.QueryOptions;

import eu.dnetlib.functionality.cql.lucene.TranslatedQuery;

import eu.dnetlib.functionality.index.utils.IndexFieldUtility;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.SolrQuery;

import org.apache.solr.common.params.ModifiableSolrParams;

import org.apache.solr.common.params.SolrParams;

/**

 * The Class SolrIndexQuery.

 * 

 * @author claudio, sandro

 */

public class SolrIndexQuery extends SolrQuery implements IndexQuery {

	/** The Constant serialVersionUID. */

	private static final long serialVersionUID = 1L;

	/**

	 * logger.

	 */

	private static final Log log = LogFactory.getLog(SolrIndexQuery.class);

	/**

	 * Instantiates a new solr index query.

	 * 

	 * @param query

	 *            the query

	 * @param options

	 *            the options

	 */

	public SolrIndexQuery(final TranslatedQuery query, final Map<String, List<String>> options) {

		this(query.asLucene(), options);

		setCqlParams(query.getOptions());

		log.debug("internal solr query: " + this.toString());

	}

	/**

	 * Instantiates a new solr index query.

	 * 

	 * @param query

	 *            the query

	 * @param options

	 *            the options

	 */

	public SolrIndexQuery(final String query, final Map<String, List<String>> options) {

		this(query);

		// TODO verify that the input options belongs to solr

		super.add(getQueryParams(options));

	}

	/**

	 * Instantiates a new solr index query.

	 * 

	 * @param query

	 *            the query

	 */

	public SolrIndexQuery(final String query) {

		super(query);

	}

	@Override

	public IndexQuery setQueryOffset(final int offset) {

		super.setStart(offset);

		return this;

	}

	@Override

	public IndexQuery setQueryLimit(final int limit) {

		super.setRows(limit);

		return this;

	}

	@Override

	public IndexQuery setQueryFacetLimit(final int limit) {

		super.setFacetLimit(limit);

		return this;

	}

	@Override

	public IndexQuery setQueryFacetOffset(final int offset) {

		super.set("facet.offset", offset);

		return this;

	}

	@Override

	public IndexQuery setDSIdFilters(final List<String> dsIds) {

		if (!isAll(dsIds)) {

			for (String id : dsIds) {

				addFilterQuery(IndexFieldUtility.DS_ID + ":\"" + id + "\"");

			}

		}

		return this;

	}

	/**

	 * Checks if is all.

	 * 

	 * @param dsIds

	 *            the ds id

	 * @return true, if is all

	 */

	protected boolean isAll(final List<String> dsIds) {

		return (dsIds != null) && (!dsIds.isEmpty()) && (dsIds.size() == 1) && dsIds.get(0).equalsIgnoreCase(IndexFieldUtility.INDEX_DSID_ALL);

	}

	/**

	 * Convert our option map to a solr option parameter map.

	 * 

	 * @param options

	 *            input paramter map.

	 * @return solr option parameter map.

	 */

	private SolrParams getQueryParams(final Map<String, List<String>> options) {

		ModifiableSolrParams params = new ModifiableSolrParams();

		String[] typeTag = new String[] {};

		for (Map.Entry<String, List<String>> entry : options.entrySet()) {

			params.add(entry.getKey(), entry.getValue().toArray(typeTag));

		}

		return params;

	}

	/**

	 * Sets the query options.

	 * 

	 * @param options

	 *            the options.

	 */

	private void setCqlParams(final QueryOptions options) {

		if (options != null) {

			if (options.getSort() != null) {

				super.addSort(options.getSort().getField(), SolrQuery.ORDER.valueOf(options.getSort().getMode().name()));

			}

		}

	}

}

package eu.dnetlib.functionality.index.client.clients.solr;

import java.io.IOException;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import eu.dnetlib.functionality.cql.CqlValueTransformerMap;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.SolrServer;

import org.apache.solr.client.solrj.SolrServerException;

import org.apache.solr.client.solrj.impl.CloudSolrServer;

import org.apache.solr.client.solrj.request.LukeRequest;

import org.apache.solr.client.solrj.response.LukeResponse;

import org.apache.solr.client.solrj.response.LukeResponse.FieldInfo;

import org.apache.solr.client.solrj.response.QueryResponse;

import com.google.common.collect.BiMap;

import com.google.common.collect.Maps;

import eu.dnetlib.data.provision.index.rmi.BrowsingRow;

import eu.dnetlib.data.provision.index.rmi.GroupResult;

import eu.dnetlib.data.provision.index.rmi.IndexServiceException;

import eu.dnetlib.functionality.index.client.AbstractIndexClient;

import eu.dnetlib.functionality.index.client.IndexClient;

import eu.dnetlib.functionality.index.client.IndexClientException;

import eu.dnetlib.functionality.index.client.response.BrowseEntry;

import eu.dnetlib.functionality.index.client.response.BrowseValueEntry;

import eu.dnetlib.functionality.index.client.response.LookupResponse;

import eu.dnetlib.functionality.index.model.Any.ValueType;

import eu.dnetlib.functionality.index.query.IndexQueryFactory;

import eu.dnetlib.functionality.index.query.QueryLanguage;

import eu.dnetlib.functionality.index.query.QueryResponseFactory;

import eu.dnetlib.functionality.index.query.QueryResponseParser;

import eu.dnetlib.functionality.index.query.SolrIndexQuery;

import eu.dnetlib.functionality.index.query.SolrIndexQueryFactory;

import eu.dnetlib.functionality.index.query.SolrIndexQueryResponse;

import eu.dnetlib.functionality.index.solr.cql.SolrTypeBasedCqlValueTransformerMapFactory;

import eu.dnetlib.functionality.index.utils.MetadataReference;

/**

 * The Class SolrIndexClient.

 */

public class SolrIndexClient extends AbstractIndexClient implements IndexClient {

	/** The server. */

	private CloudSolrServer server;

	private SolrIndexQueryFactory solrIndexQueryFactory;

	/** The query response factory. */

	private QueryResponseFactory<QueryResponse> queryResponseFactory;

	private SolrTypeBasedCqlValueTransformerMapFactory tMapFactory;

	private static final Log log = LogFactory.getLog(SolrIndexClient.class);

	/**

	 * The Constructor.

	 *

	 * @param format

	 *            the format

	 * @param layout

	 *            the layout

	 * @param interpretation

	 *            the interpretation

	 * @param serviceProperties

	 *            the service properties

	 * @param tMapFactory

	 */

	public SolrIndexClient(final String format, final String layout, final String interpretation, final Map<String, String> serviceProperties,

			final SolrIndexQueryFactory indexQueryFactory, final QueryResponseFactory<QueryResponse> queryResponseFactory,

			final SolrTypeBasedCqlValueTransformerMapFactory tMapFactory) {

		super(format, layout, interpretation, serviceProperties);

		log.debug(String.format("Created a new instance of the index of type %s-%s-%s", format, layout, interpretation));

		this.solrIndexQueryFactory = indexQueryFactory;

		this.queryResponseFactory = queryResponseFactory;

		this.tMapFactory = tMapFactory;

	}

	/**

	 * Do delete.

	 *

	 * @param query

	 *            the CQL query

	 * @return true, if do delete

	 * @throws IndexServiceException

	 *             the index service exception

	 */

	@Override

	public long delete(final String query) throws IndexClientException {

		try {

			log.debug("delete by query: " + query);

			MetadataReference mdRef = new MetadataReference(getFormat(), getLayout(), getInterpretation());

			SolrIndexQuery translatedQuery = (SolrIndexQuery) solrIndexQueryFactory.getIndexQuery(QueryLanguage.CQL, query, this, mdRef);

			String tquery = translatedQuery.getQuery();

			translatedQuery.setQueryLimit(0);

			SolrIndexQueryResponse rsp = new SolrIndexQueryResponse(server.query(translatedQuery));

			QueryResponseParser responseParser = queryResponseFactory.getQueryResponseParser(rsp, mdRef);

			long total = responseParser.getNumFound();

			server.deleteByQuery(tquery);

			server.commit();

			return total;

		} catch (Exception e) {

			throw new IndexClientException("unable to run delete by query: " + query, e);

		}

	}

	/**

	 * {@inheritDoc}

	 *

	 * @throws IndexClientException

	 *

	 * @see eu.dnetlib.functionality.index.client.IndexClient#browse(String, List, int)

	 */

	@Override

	public List<BrowseEntry> browse(final String query, final List<String> browseFields, final int max) throws IndexClientException {

		MetadataReference mdRef = new MetadataReference(getFormat(), getLayout(), getInterpretation());

		SolrIndexQuery translatedQuery = buildBrowseQuery(query, browseFields, max, mdRef);

		return executeBrowseQuery(query, translatedQuery, mdRef, browseFields);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @throws IndexClientException

	 *

	 * @see eu.dnetlib.functionality.index.client.IndexClient#browse(String, List, int, List)

	 */

	@Override

	public List<BrowseEntry> browse(final String query, final List<String> browseFields, final int max, final List<String> filterQuery)

			throws IndexClientException {

		MetadataReference mdRef = new MetadataReference(getFormat(), getLayout(), getInterpretation());

		SolrIndexQuery translatedQuery = buildBrowseQuery(query, browseFields, max, mdRef);

		if (filterQuery != null) {

			log.debug("Filter Query:");

			for (String fq : filterQuery) {

				translatedQuery.addFilterQuery(fq);

				log.debug("- " + fq);

			}

		}

		return executeBrowseQuery(query, translatedQuery, mdRef, browseFields);

	}

	public SolrIndexQuery buildBrowseQuery(final String query, final List<String> browseFields, final int max, final MetadataReference mdRef)

			throws IndexClientException {

		log.debug("Browse request for the index collection for query:" + query);

		SolrIndexQuery translatedQuery = (SolrIndexQuery) solrIndexQueryFactory.getIndexQuery(QueryLanguage.CQL, query, this, mdRef);

		translatedQuery.setFacet(true);

		if (browseFields != null) {

			List<String> browsableFields = solrIndexQueryFactory.getBrowsableFields(browseFields, mdRef);

			log.debug("Browsing fields:");

			for (String field : browsableFields) {

				translatedQuery.addFacetField(field);

				log.debug("- " + field);

			}

			translatedQuery.setFacetLimit(max);

			log.debug("max number of browsing field :" + max);

		}

		return translatedQuery;

	}

	private List<BrowseEntry> executeBrowseQuery(final String originalQuery,

			final SolrIndexQuery query,

			final MetadataReference mdRef,

			final List<String> browseFields) throws IndexClientException {

		try {

			SolrIndexQueryResponse response = new SolrIndexQueryResponse(server.query(query));

			QueryResponseParser responseParser = queryResponseFactory.getQueryResponseParser(response, mdRef);

			List<BrowsingRow> results = responseParser.getBrowsingResults();

			List<BrowseEntry> out = convertBrowseEntry(browseFields, results, responseParser.getAliases());

			return out;

		} catch (SolrServerException e) {

			throw new IndexClientException("Error on executing a query " + originalQuery, e);

		}

	}

	/**

	 * Creates the connection.

	 *

	 * @return the string

	 * @throws IndexClientException

	 *             the index client exception

	 */

	private String getUrl() throws IndexClientException {

		String address = serviceProperties.get(ZK_ADDRESS);

		if (address == null) { throw new IndexClientException("Unable to load a solr client, missing zk address"); }

		return address;

	}

	/**

	 * Gets the server.

	 *

	 * @return the server

	 * @throws IndexClientException

	 *             the index client exception

	 */

	public SolrServer getServer() throws IndexClientException {

		if (this.server == null) {

			String url = getUrl();

			log.debug("create new Client " + url);

			server = new CloudSolrServer(url);

			server.setDefaultCollection(String.format("%s-%s-%s", getFormat(), getLayout(), getInterpretation()));

			try {

				server.ping();

			} catch (Exception e) {

				throw new IndexClientException("oops something went wrong", e);

			}

		}

		return server;

	}

	/**

	 * Sets the server.

	 *

	 * @param server

	 *            the server

	 */

	public void setServer(final CloudSolrServer server) {

		this.server = server;

	}

	@Override

	public LookupResponse lookup(final String query, final List<String> filterQuery, final int from, final int to) throws IndexClientException {

		log.debug("lookup request for the index collection for query:" + query);

		MetadataReference mdRef = new MetadataReference(getFormat(), getLayout(), getInterpretation());

		SolrIndexQuery translatedQuery = (SolrIndexQuery) solrIndexQueryFactory.getIndexQuery(QueryLanguage.CQL, query, this, mdRef);

		translatedQuery.setQueryOffset(from);

		translatedQuery.setQueryLimit(to - from + 1);

		if (filterQuery != null) {

			for (String fq : filterQuery) {

				translatedQuery.addFilterQuery(fq);

			}

		}

		try {

			SolrIndexQueryResponse response = new SolrIndexQueryResponse(server.query(translatedQuery));

			QueryResponseParser responseParser = queryResponseFactory.getQueryResponseParser(response, mdRef);

			return new LookupResponse(responseParser);

		} catch (SolrServerException e) {

			throw new IndexClientException("Error on executing a query " + query, e);

		}

	}

	@Override

	public CqlValueTransformerMap getCqlValueTransformerMap(final MetadataReference mdRef) throws IndexClientException {

		try {

			return this.tMapFactory.getIt(readFieldNamesAndTypes(mdRef.toString()));

		} catch (Exception e) {

			throw new IndexClientException(e);

		}

	}

	@Override

	public IndexQueryFactory getIndexQueryFactory() {

		return solrIndexQueryFactory;

	}

	@Override

	public void stop() throws IndexClientException {

		try {

			log.debug("shutdown client: " + serviceProperties.get(ZK_ADDRESS));

			server.shutdown();

		} catch (Throwable e) {

			throw new IndexClientException(e);

		}

	}

	private List<BrowseEntry> convertBrowseEntry(final List<String> browseFields, final List<BrowsingRow> results, final BiMap<String, String> aliases) {

		Map<String, BrowseEntry> mapResult = new HashMap<String, BrowseEntry>();

		for (BrowsingRow row : results) {

			for (GroupResult groupResult : row.getGroupResult()) {

				String name = groupResult.getName();

				List<BrowseValueEntry> valuesEntry;

				BrowseEntry entry;

				if (mapResult.containsKey(name)) {

					entry = mapResult.get(name);

					valuesEntry = entry.getValues();

					if (valuesEntry == null) {

						valuesEntry = new ArrayList<BrowseValueEntry>();

						entry.setValues(valuesEntry);

					}

				} else {

					entry = new BrowseEntry();

					entry.setField(name);

					entry.setLabel(name);

					valuesEntry = new ArrayList<BrowseValueEntry>();

					entry.setValues(valuesEntry);

					mapResult.put(name, entry);

				}

				String value = groupResult.getValue();

				int count = groupResult.getCount();

				BrowseValueEntry entryValue = new BrowseValueEntry(value, count);

				valuesEntry.add(entryValue);

			}

		}

		List<BrowseEntry> out = new ArrayList<BrowseEntry>();

		for (String b : browseFields) {

			String inverse = null;

			if (aliases != null) {

				inverse = aliases.get(b) != null ? aliases.get(b) : aliases.inverse().get(b);

			}

			if (mapResult.containsKey(b)) {

				out.add(mapResult.get(b));

			} else if (mapResult.containsKey(inverse) == true) {

				BrowseEntry data = mapResult.get(inverse);

				data.setField(b);

				out.add(data);

			}

		}

		return out;

	}

	private Map<String, ValueType> readFieldNamesAndTypes(final String coreName) throws SolrServerException, IOException, IndexClientException {

		// final SolrServer server = cloudServer.getSolrServer(coreName);

		final LukeRequest request = new LukeRequest();

		request.setShowSchema(true);

		// cloudServer.setDefaultCollection(coreName);

		request.setNumTerms(0);

		final LukeResponse response = request.process(getServer());

		final Map<String, FieldInfo> fieldInfos = response.getFieldInfo();

		final Map<String, LukeResponse.FieldTypeInfo> fieldTypeInfos = response.getFieldTypeInfo();

		final Map<String, ValueType> result = Maps.newHashMap();

		for (FieldInfo fieldInfo : fieldInfos.values()) {

			LukeResponse.FieldTypeInfo fieldTypeInfo = fieldTypeInfos.get(fieldInfo.getType());

			final String fieldName = fieldTypeInfo.getName().toLowerCase();

			final ValueType fieldType = resolveSolrTypeClassName(fieldName);

			result.put(fieldInfo.getName(), fieldType);

		}

		return result;

	}

	private ValueType resolveSolrTypeClassName(final String solrTypeName) {

		if (solrTypeName.contains("LongField")) {

			return ValueType.LONG;

		} else if (solrTypeName.contains("IntField")) {

			return ValueType.LONG;

		} else if (solrTypeName.contains("short")) {

			return ValueType.LONG;

		} else if (solrTypeName.contains("float")) {

			return ValueType.DOUBLE;

		} else if (solrTypeName.contains("double")) {

			return ValueType.DOUBLE;

		} else if (solrTypeName.contains("date")) {

			return ValueType.DATETIME;

		} else {

			return ValueType.STRING;

		}

	}

}

package eu.dnetlib.functionality.index.client.clients.solr;

import java.lang.reflect.Type;

import java.util.Map;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

import com.google.common.reflect.TypeToken;

import com.google.gson.Gson;

import eu.dnetlib.functionality.index.client.AbstractIndexClientFactory;

import eu.dnetlib.functionality.index.client.IndexClient;

import eu.dnetlib.functionality.index.client.IndexClientException;

import eu.dnetlib.functionality.index.client.IndexClientFactory;

import eu.dnetlib.functionality.index.query.SolrIndexQueryFactory;

import eu.dnetlib.functionality.index.query.SolrIndexQueryResponseFactory;

import eu.dnetlib.functionality.index.solr.cql.SolrTypeBasedCqlValueTransformerMapFactory;

/**

 * The Class SolrIndexClientFactory.

 */

public class SolrIndexClientFactory extends AbstractIndexClientFactory implements IndexClientFactory {

	/** The id. */

	private static String ID = "id";

	/** The configuration. */

	private String configuration;

	/** The type token. */

	@SuppressWarnings("serial")

	protected Type typeToken = new TypeToken<Map<String, String>>() {}.getType();

	/** The service properties. */

	private Map<String, String> serviceProperties;

	@Autowired

	private SolrIndexQueryFactory solrIndexQueryFactory;

	@Autowired

	private SolrIndexQueryResponseFactory solrIndexQueryResponseFactory;

	@Autowired

	private SolrTypeBasedCqlValueTransformerMapFactory tMapFactory;

	/**

	 * Inits the class.

	 *

	 * @throws IndexClientException

	 *             the index client exception

	 */

	@Override

	public void init() throws IndexClientException {

		try {

			serviceProperties = new Gson().fromJson(getConfiguration(), typeToken);

		} catch (Throwable e) {

			throw new IndexClientException("unable to parse configuration: " + getConfiguration(), e);

		}

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.client.IndexClientFactory#getBackendId()

	 */