D-Net

package eu.dnetlib.functionality.index.solr.cql;

import java.util.Map;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.functionality.index.model.Any.ValueType;

import eu.dnetlib.miscutils.functional.UnaryFunction;

/**

 * Factory for the SolrTypeBasedCqlValueTransformerMap class objects

 * 

 * @author claudio

 * 

 */

public class SolrTypeBasedCqlValueTransformerMapFactory {

	/**

	 * Map of functions, injected via spring.

	 */

	private Map<String, UnaryFunction<String, String>> transformerMap;

	/**

	 * Method returns a new instance of SolrTypeBasedCqlValueTransformerMap.

	 * 

	 * @param schema

	 * @return

	 */

	public SolrTypeBasedCqlValueTransformerMap getIt(final Map<String, ValueType> schema) {

		return new SolrTypeBasedCqlValueTransformerMap(schema, getTransformerMap());

	}

	@Required

	public void setTransformerMap(Map<String, UnaryFunction<String, String>> transformerMap) {

		this.transformerMap = transformerMap;

	}

	public Map<String, UnaryFunction<String, String>> getTransformerMap() {

		return transformerMap;

	}

}

package eu.dnetlib.functionality.index.solr.cql;

import eu.dnetlib.miscutils.functional.UnaryFunction;

/**

 * Simply and not very roboust normalizer for solr dates. Basically it handles well yyyy-mm-dd and

 * yyyy-mm-ddThh:mm:ssZ

 * 

 * @author marko

 * 

 */

public class SimpleDateValueTransformer implements UnaryFunction<String, String> {

	@Override

	public String evaluate(final String value) {

		if (!value.endsWith("Z"))

			return value + "T00:00:00Z";

		return value;

	}

}

package eu.dnetlib.functionality.index.solr.cql;

import java.util.Map;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.common.SolrException;

import eu.dnetlib.functionality.index.cql.CqlValueTransformerMap;

import eu.dnetlib.functionality.index.model.Any.ValueType;

import eu.dnetlib.miscutils.functional.IdentityFunction;

import eu.dnetlib.miscutils.functional.UnaryFunction;

/**

 * This class maps the fields in the given index schema with a transformation rule.

 * 

 * @author marko

 * 

 */

public class SolrTypeBasedCqlValueTransformerMap implements CqlValueTransformerMap {

	/**

	 * logger.

	 */

	private static final Log log = LogFactory.getLog(SolrTypeBasedCqlValueTransformerMap.class); // NOPMD by marko on 11/24/08 5:02 PM

	/**

	 * Index schema.

	 */

	private final Map<String, ValueType> schema;

	/**

	 * Map of functions.

	 */

	private final Map<String, UnaryFunction<String, String>> transformerMap;

	/**

	 * Create value transformer map bound to a specific schema

	 * 

	 * @param indexSchema

	 */

	public SolrTypeBasedCqlValueTransformerMap(final Map<String, ValueType> schema, final Map<String, UnaryFunction<String, String>> transformerMap) {

		this.schema = schema;

		this.transformerMap = transformerMap;

	}

	/**

	 * {@inheritDoc}

	 * 

	 * @see eu.dnetlib.functionality.index.cql.CqlValueTransformerMap#transformerFor(java.lang.String)

	 */

	@Override

	public UnaryFunction<String, String> transformerFor(final String fieldName) {

		try {

			final ValueType field = schema.get(fieldName);

			if (field != null) {

				UnaryFunction<String, String> res = transformerMap.get(field.name());

				if (res != null) {

					return res;

				}

			}

		} catch (SolrException e) {

			log.debug("cannot find field", e);

		}

		return new IdentityFunction<String>();

	}

}

package eu.dnetlib.functionality.index.solr.utils;

import org.apache.oro.text.perl.Perl5Util;

import eu.dnetlib.miscutils.functional.UnaryFunction;

public class HighlightUtils implements UnaryFunction<String, String> {

	public final static String DEFAULT_HL_PRE = "[hl]";

	public final static String DEFAULT_HL_POST = "[/hl]";

	private static String CLEAN_HEADER = "s#\\[/?hl\\]##gm";

	private static String CLEAN_REGEX_OPEN = "<([^>]*)\\[hl\\]([^>]*)>";

	private static String CLEAN_REGEX_CLOSE = "<([^>]*)\\[\\/hl\\]([^>]*)>";

	// private static String CLEAN_REGEX_OPEN = "s#<([^>]*)\\[hl\\]([^>]*)>#<$1$2>#gm";

	// private static String CLEAN_REGEX_CLOSE = "s#<([^>]*)\\[\\/hl\\]([^>]*)>#<$1$2>#gm";

	private Perl5Util p5util = new Perl5Util();

	@Override

	public String evaluate(final String doc) {

		String[] chunk = doc.split("</header>");

		String string = cleanHeader(chunk[0]) + "</header>" + cleanBody(chunk[1]);

		return string;

	}

	private String cleanHeader(final String header) {

		return p5util.substitute(CLEAN_HEADER, header);

	}

	// TODO: implement a faster way to do this

	private String cleanBody(final String body) {

		String res = body.replaceAll(CLEAN_REGEX_OPEN, "<$1$2>").replaceAll(CLEAN_REGEX_CLOSE, "<$1$2>");

		if (res.equals(body)) return res;

		return cleanBody(res);

	}

}

package eu.dnetlib.functionality.index.solr.feed;

import java.io.IOException;

import java.net.MalformedURLException;

import java.net.URI;

import java.net.URL;

import java.util.Collection;

import java.util.Iterator;

import java.util.List;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.SolrServer;

import org.apache.solr.client.solrj.SolrServerException;

import org.apache.solr.client.solrj.impl.CloudSolrServer;

import org.apache.solr.client.solrj.impl.HttpSolrServer;

import org.apache.solr.client.solrj.response.UpdateResponse;

import org.apache.solr.common.SolrInputDocument;

import com.google.common.base.Splitter;

import com.google.common.collect.Lists;

import com.google.common.hash.HashFunction;

import com.google.common.hash.Hashing;

import eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions;

@Deprecated

public class SolrServerPool {

	/**

	 * logger.

	 */

	private static final Log log = LogFactory.getLog(SolrServerPool.class); // NOPMD by marko on 11/24/08 5:02 PM

	/*

	 * We run into problems when using the ConcurrentUpdateSolrServer, so we're sticking to the HttpSolrServer.

	 */

	private final List<HttpSolrServer> updateServerPool = Lists.newArrayList();

	private CloudSolrServer cloudServer;

	private final HashFunction hash = Hashing.murmur3_32();

	public SolrServerPool(final String updateUrlLocal, final String updateUrlList, final String zkHost, final String collection, final boolean localFeeding) {

		for (URL url : parseUrlListPattern(updateUrlLocal, updateUrlList, localFeeding)) {

			updateServerPool.add(new HttpSolrServer(url + "/" + collection));

		}

		cloudServer = new CloudSolrServer(zkHost);

		cloudServer.setDefaultCollection(collection);

	}

	public UpdateResponse add(final SolrInputDocument doc) throws SolrServerException, IOException {

		return updateServerPool.get(hashPick(doc)).add(doc);

	}

	public UpdateResponse addAll(final Iterator<SolrInputDocument> docs) throws SolrServerException, IOException {

		if (updateServerPool.size() == 1) return updateServerPool.get(0).add(docs);

		int i = Integer.parseInt(DnetXsltFunctions.randomInt(updateServerPool.size()));

		return updateServerPool.get(i).add(docs);

	}

	public UpdateResponse addAll(final Collection<SolrInputDocument> docs) throws SolrServerException, IOException {

		if (updateServerPool.size() == 1) return updateServerPool.get(0).add(docs);

		int i = Integer.parseInt(DnetXsltFunctions.randomInt(updateServerPool.size()));

		return updateServerPool.get(i).add(docs);

	}

	public void deleteByQuery(final String query) throws SolrServerException, IOException {

		cloudServer.deleteByQuery(query);

	}

	public void commitAll() throws SolrServerException, IOException {

		cloudServer.commit();

	}

	public void shutdownAll() throws SolrServerException {

		cloudServer.shutdown();

		for (SolrServer server : updateServerPool) {

			server.shutdown();

		}

	}

	// //////////////////

	private int hashPick(final SolrInputDocument doc) {

		final int hashCode = hash.hashBytes(doc.getFieldValue("__indexrecordidentifier").toString().getBytes()).asInt();

		return Math.abs(hashCode) % updateServerPool.size();

	}

	public List<URL> parseUrlListPattern(final String local, final String list, final boolean localFeeding) {

		final List<URL> res = Lists.newArrayList();

		try {

			if (localFeeding) {

				res.add(new URL(local));

			} else {

				Matcher matcher = Pattern.compile("(^.*)\\[(\\d+)\\.\\.(\\d+)\\](.*$)").matcher(list);

				if (matcher.matches()) {

					final String prefix = matcher.group(1);

					int lb = Integer.parseInt(matcher.group(2));

					int ub = Integer.parseInt(matcher.group(3));

					final String suffix = matcher.group(4);

					for (int i = lb; i <= ub; i++) {

						res.add(new URL(prefix + i + suffix));

					}

				}

			}

		} catch (MalformedURLException e) {

			throw new IllegalArgumentException("invalid url list: " + list, e);

		}

		log.info("parsed url(s): " + res);

		return res;

	}

	public List<URL> parseUrlList(final String list) throws MalformedURLException {

		final List<URL> res = Lists.newArrayList();

		for (final String url : Splitter.on(",").trimResults().split(list)) {

			res.add(URI.create(url).toURL());

		}

		return res;

	}

}

package eu.dnetlib.functionality.index.solr.feed;

import java.io.StringReader;

import java.io.StringWriter;

import java.util.ArrayList;

import javax.xml.stream.XMLEventFactory;

import javax.xml.stream.XMLEventReader;

import javax.xml.stream.XMLEventWriter;

import javax.xml.stream.XMLInputFactory;

import javax.xml.stream.XMLOutputFactory;

import javax.xml.stream.XMLStreamException;

import javax.xml.stream.events.Namespace;

import javax.xml.stream.events.StartElement;

import javax.xml.stream.events.XMLEvent;

import org.apache.solr.common.SolrInputDocument;

import org.dom4j.DocumentException;

import com.google.common.collect.Lists;

/**

 * Optimized version of the document parser, drop in replacement of InputDocumentFactory.

 *

 * <p>

 * Faster because:

 * </p>

 * <ul>

 * <li>Doesn't create a DOM for the full document</li>

 * <li>Doesn't execute xpaths agains the DOM</li>

 * <li>Quickly serialize the 'result' element directly in a string.</li>

 * <li>Uses less memory: less pressure on GC and allows more threads to process this in parallel</li>

 * </ul>

 *

 * <p>

 * This class is fully reentrant and can be invoked in parallel.

 * </p>

 *

 * @author marko

 *

 */

public class StreamingInputDocumentFactory extends InputDocumentFactory {

	protected static final String DNETRESULT = "result";

	protected static final String TARGETFIELDS = "targetFields";

	protected static final String INDEX_RECORD_ID_ELEMENT = "indexRecordIdentifier";

	protected ThreadLocal<XMLInputFactory> inputFactory = new ThreadLocal<XMLInputFactory>() {

		@Override

		protected XMLInputFactory initialValue() {

			return XMLInputFactory.newInstance();

		}

	};

	protected ThreadLocal<XMLOutputFactory> outputFactory = new ThreadLocal<XMLOutputFactory>() {

		@Override

		protected XMLOutputFactory initialValue() {

			return XMLOutputFactory.newInstance();

		}

	};

	protected ThreadLocal<XMLEventFactory> eventFactory = new ThreadLocal<XMLEventFactory>() {

		@Override

		protected XMLEventFactory initialValue() {

			return XMLEventFactory.newInstance();

		}

	};

	/**

	 * {@inheritDoc}

	 *

	 * @throws XMLStreamException

	 * @throws DocumentException

	 *

	 * @see eu.dnetlib.functionality.index.solr.feed.InputDocumentFactory#parseDocument(eu.dnetlib.functionality.index.solr.feed.IndexDocument,

	 *      java.lang.String)

	 */

	@Override

	public SolrInputDocument parseDocument(final String version, final String inputDocument, final String dsId) {

		final StringWriter results = new StringWriter();

		try {

			XMLEventReader parser = inputFactory.get().createXMLEventReader(new StringReader(inputDocument));

			final SolrInputDocument indexDocument = new SolrInputDocument();

			while (parser.hasNext()) {

				final XMLEvent event = parser.nextEvent();

				if ((event != null) && event.isStartElement()) {

					final String localName = event.asStartElement().getName().getLocalPart();

					if (INDEX_RECORD_ID_ELEMENT.equals(localName)) {

						final XMLEvent text = parser.nextEvent();

						String recordId = getText(text);

						indexDocument.addField(INDEX_RECORD_ID, recordId);

					} else if (TARGETFIELDS.equals(localName)) {

						parseTargetFields(indexDocument, parser);

					} else if (DNETRESULT.equals(localName)) {

						copyResult(indexDocument, results, parser);

					}

				}

			}

			if (version != null) {

				indexDocument.addField(DS_VERSION, version);

			}

			if (dsId != null) {

				indexDocument.addField(DS_ID, dsId);

			}

			if (!indexDocument.containsKey(INDEX_RECORD_ID)) {

				indexDocument.clear();

				System.err.println("missing indexrecord id:\n" + inputDocument);

			}

			return indexDocument;

		} catch (XMLStreamException e) {

			return new SolrInputDocument();

		}

	}

	/**

	 * Parse the targetFields block and add fields to the solr document.

	 *

	 * @param indexDocument

	 * @param parser

	 * @throws XMLStreamException

	 */

	protected void parseTargetFields(final SolrInputDocument indexDocument, final XMLEventReader parser) throws XMLStreamException {

		boolean hasFields = false;

		while (parser.hasNext()) {

			final XMLEvent targetEvent = parser.nextEvent();

			if (targetEvent.isEndElement() && targetEvent.asEndElement().getName().getLocalPart().equals(TARGETFIELDS)) {

				break;

			}

			if (targetEvent.isStartElement()) {

				final String fieldName = targetEvent.asStartElement().getName().getLocalPart();

				final XMLEvent text = parser.nextEvent();

				String data = getText(text);

				addField(indexDocument, fieldName, data);

				hasFields = true;

			}

		}

		if (!hasFields) {

			indexDocument.clear();

		}

	}

	/**

	 * Copy the /indexRecord/result element and children, preserving namespace declarations etc.

	 *

	 * @param indexDocument

	 * @param results

	 * @param parser

	 * @throws XMLStreamException

	 */

	protected void copyResult(final SolrInputDocument indexDocument, final StringWriter results, final XMLEventReader parser) throws XMLStreamException {

		final XMLEventWriter writer = outputFactory.get().createXMLEventWriter(results);

		// TODO: newRecord should copy all the namespace prefixes setup in parents

		// fortunately the only parent of the result element is the 'indexrecord', so it should be easy to get

		// the namespaces declared on the root element (and fast)

		final ArrayList<Namespace> namespaces = Lists.newArrayList(eventFactory.get().createNamespace("dri", "http://www.driver-repository.eu/namespace/dri"),

				eventFactory.get().createNamespace("dr", "http://www.driver-repository.eu/namespace/dr"),

				eventFactory.get().createNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance"),

				eventFactory.get().createNamespace("dc", "http://purl.org/dc/elements/1.1/"),

				eventFactory.get().createNamespace("oaf", "http://namespace.openaire.eu/oaf"));

		StartElement newRecord = eventFactory.get().createStartElement("", null, RESULT, null, namespaces.iterator());

		// new root record

		writer.add(newRecord);

		// copy the rest as it is

		while (parser.hasNext()) {

			final XMLEvent resultEvent = parser.nextEvent();

			// TODO: replace with depth tracking instead of close tag tracking.

			if (resultEvent.isEndElement() && resultEvent.asEndElement().getName().getLocalPart().equals(DNETRESULT)) {

				writer.add(eventFactory.get().createEndElement("", null, RESULT));

				break;

			}

			writer.add(resultEvent);

		}

		writer.close();

		indexDocument.addField(INDEX_RESULT, results.toString());

	}

	/**

	 * Helper used to add a field to a solr doc. It avoids to add empy fields

	 *

	 * @param indexDocument

	 * @param field

	 * @param value

	 */

	private final void addField(final SolrInputDocument indexDocument, final String field, final String value) {

		String cleaned = value.trim();

		if (!cleaned.isEmpty()) {

			// log.info("\n\n adding field " + field.toLowerCase() + " value: " + cleaned + "\n");

			indexDocument.addField(field.toLowerCase(), cleaned);

		}

	}

	/**

	 * Helper used to get the string from a text element.

	 *

	 * @param text

	 * @return

	 */

	protected final String getText(final XMLEvent text) {

		if (text.isEndElement()) // log.warn("skipping because isEndOfElement " + text.asEndElement().getName().getLocalPart());

			return "";

		return text.asCharacters().getData();

	}

}

package eu.dnetlib.functionality.index.solr.feed;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.Arrays;

import java.util.List;

import javax.xml.stream.XMLStreamException;

import org.apache.solr.common.SolrInputDocument;

/**

 * 

 * @author claudio

 * 

 */

public abstract class InputDocumentFactory {

	public static final String INDEX_FIELD_PREFIX = "__";

	public static final String DS_VERSION = INDEX_FIELD_PREFIX + "dsversion";

	public static final String DS_ID = INDEX_FIELD_PREFIX + "dsid";

	public static final String RESULT = "result";

	public static final String INDEX_RESULT = INDEX_FIELD_PREFIX + RESULT;

	public static final String INDEX_RECORD_ID = INDEX_FIELD_PREFIX + "indexrecordidentifier";

	private static final String outFormat = new String("yyyy-MM-dd'T'hh:mm:ss'Z'");

	private final static List<String> dateFormats = Arrays.asList("yyyy-MM-dd'T'hh:mm:ss", "yyyy-MM-dd", "dd-MM-yyyy", "dd/MM/yyyy", "yyyy");

	public abstract SolrInputDocument parseDocument(final String version, String inputDocument, String dsId) throws XMLStreamException;

	/**

	 * method return a solr-compatible string representation of a date

	 * 

	 * @param date

	 * @return

	 * @throws DocumentException

	 * @throws ParseException

	 */

	public static String getParsedDateField(final String date) {

		for (String formatString : dateFormats) {

			try {

				return new SimpleDateFormat(outFormat).format(new SimpleDateFormat(formatString).parse(date));

			} catch (ParseException e) {

			}

		}

		throw new IllegalStateException("unable to parse date: " + date);

	}

	public String parseDate(final String date) {

		return getParsedDateField(date);

	}

}

package eu.dnetlib.functionality.index.solr.feed;

import java.util.Map;

import javax.xml.stream.XMLStreamException;

import org.springframework.beans.factory.annotation.Required;

import com.google.common.base.Function;

import eu.dnetlib.functionality.index.feed.DocumentMapperFactory;

import eu.dnetlib.functionality.index.model.Any.ValueType;

import eu.dnetlib.functionality.index.model.document.IndexDocument;

import eu.dnetlib.functionality.index.model.document.Status;

import eu.dnetlib.functionality.index.model.util.SolrIndexDocument;

import eu.dnetlib.functionality.index.utils.MetadataReference;

/**

 * A factory for creating SolrDocumentMapper objects.

 */

public class SolrDocumentMapperFactory implements DocumentMapperFactory {

	/**

	 * document factory used for the feed process.

	 */

	private InputDocumentFactory documentFactory;

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.feed.DocumentMapperFactory#getRecordMapper(java.util.Map,

	 *      eu.dnetlib.functionality.index.utils.MetadataReference, java.lang.String, java.lang.String)

	 */

	@Override

	public Function<String, IndexDocument> getRecordMapper(final Map<String, ValueType> schema,

			final MetadataReference mdRef,

			final String dsId,

			final String version) {

		return new Function<String, IndexDocument>() {

			@Override

			public IndexDocument apply(final String doc) {

				SolrIndexDocument indexDocument = new SolrIndexDocument(schema, dsId);

				try {

					indexDocument.setContent(documentFactory.parseDocument(version, doc, dsId));

				} catch (XMLStreamException e) {

					return indexDocument.setMarked();

				}

				indexDocument.setStatus(Status.OK);

				return indexDocument;

			}

		};

	}

	/**

	 * Gets the document factory.

	 *

	 * @return the documentFactory

	 */

	public InputDocumentFactory getDocumentFactory() {

		return documentFactory;

	}

	/**

	 * Sets the document factory.

	 *

	 * @param documentFactory

	 *            the documentFactory to set

	 */

	@Required

	public void setDocumentFactory(final InputDocumentFactory documentFactory) {

		this.documentFactory = documentFactory;

	}

}

package eu.dnetlib.functionality.index.model.util;

import java.util.Iterator;

import java.util.Map.Entry;

import org.apache.solr.common.SolrDocumentList;

import org.apache.solr.common.util.NamedList;

import eu.dnetlib.functionality.index.model.AnyMap;

import eu.dnetlib.functionality.index.model.DataFactory;

import eu.dnetlib.functionality.index.model.InvalidValueTypeException;

import eu.dnetlib.functionality.index.model.Value;

import eu.dnetlib.functionality.index.model.impl.DefaultDataFactoryImpl;

/**

 * The Class AnySolrUtil.

 */

public class AnySolrUtil extends AnyUtil {

	/**

	 * Convert named list to any map.

	 * 

	 * @param list

	 *            the list

	 * @param map

	 *            the map

	 * @return the any map

	 */

	@SuppressWarnings("unchecked")

	public static AnyMap convertNamedListToAnyMap(final NamedList<Object> list, final AnyMap map) {

		final Iterator<Entry<String, Object>> it = list.iterator();

		while (it.hasNext()) {

			Entry<String, Object> entry = it.next();

			final String key = entry.getKey();

			final Object obj = entry.getValue();

			if (obj instanceof NamedList<?>) {

				final AnyMap subMap = map.getMap(key, true);

				convertNamedListToAnyMap((NamedList<Object>) obj, subMap);

			} else if (obj instanceof SolrDocumentList) {

				SolrDocumentList docList = (SolrDocumentList) obj;

				AnyMap response = DataFactory.DEFAULT.createAnyMap();

				response.put("numFound", docList.getNumFound());

				response.put("start", docList.getStart());

				response.put("maxScore", docList.getMaxScore());

				response.put("docs", objectToAny(obj));

				map.put("response", response);

			} else {

				try {

					final Value value = DataFactory.DEFAULT.autoConvertValue(obj);

					map.put(key, value);

				} catch (InvalidValueTypeException exception) {

					; // skip

				}

			}

		}

		return map;

	}

	/**

	 * Convert named list to any map.

	 * 

	 * @param list

	 *            the list

	 * @return the any map

	 */

	public static AnyMap convertNamedListToAnyMap(final NamedList<Object> list) {

		return convertNamedListToAnyMap(list, DefaultDataFactoryImpl.INSTANCE.createAnyMap());

	}

}

package eu.dnetlib.functionality.index.model.util;

import java.util.HashMap;

import java.util.Map;

import org.apache.solr.common.SolrInputDocument;

import org.apache.solr.common.SolrInputField;

import eu.dnetlib.functionality.index.model.Any.ValueType;

import eu.dnetlib.functionality.index.model.document.AbstractIndexDocument;

// TODO: Auto-generated Javadoc

/**

 * The Class SolrIndexDocument an implementation of the index document for SOLR.

 */

public class SolrIndexDocument extends AbstractIndexDocument {

	/**

	 * Instantiates a new solr index document.

	 *

	 * @param schema

	 *            the schema

	 * @param dsId

	 *            the ds id

	 */

	public SolrIndexDocument(final Map<String, ValueType> schema, final String dsId) {

		super(schema, dsId);

	}

	/**

	 * Instantiates a new solr index document.

	 *

	 * @param schema

	 *            the schema

	 * @param dsId

	 *            the ds id

	 * @param solrDocument

	 *            the solr document

	 */

	public SolrIndexDocument(final Map<String, ValueType> schema, final String dsId, final SolrInputDocument solrDocument) {

		super(schema, dsId);

		addFields(solrDocument);

	}

	/**

	 * Adds the fields.

	 *

	 * @param solrDocument

	 *            the solr document

	 */

	private void addFields(final SolrInputDocument solrDocument) {

		for (String name : solrDocument.getFieldNames()) {

			Object fieldValue = solrDocument.getFieldValue(name);

			addField(name, fieldValue);

		}

	}

	/**

	 * Sets the content.

	 *

	 * @param solrDocument

	 *            the new content

	 */

	public void setContent(final SolrInputDocument solrDocument) {

		addFields(solrDocument);

	}

	/**

	 * Gets the solr document.

	 *

	 * @return the solr document

	 */

	public SolrInputDocument getSolrDocument() {

		Map<String, SolrInputField> data = new HashMap<String, SolrInputField>();

		for (String key : fields.keySet()) {

			SolrInputField solrField = new SolrInputField(key);

			for (Object o : fields.get(key)) {

				solrField.addValue(o, 1.0f);

			}

			data.put(key, solrField);

		}

		return new SolrInputDocument(data);

	}

}

package eu.dnetlib.functionality.index.query;

import org.apache.solr.client.solrj.response.QueryResponse;

import eu.dnetlib.functionality.index.client.IndexClientException;

import eu.dnetlib.functionality.index.utils.MetadataReference;

/**

 * The Class SolrIndexQueryResponseFactory.

 */

public class SolrIndexQueryResponseFactory extends QueryResponseFactory<QueryResponse> {

	/**

	 * {@inheritDoc}

	 * 

	 * @throws IndexClientException

	 *

	 * @see eu.dnetlib.functionality.index.query.QueryResponseFactory#getQueryResponseParser(eu.dnetlib.functionality.index.query.IndexQueryResponse,

	 *      eu.dnetlib.functionality.index.utils.MetadataReference)

	 */

	@Override

	public QueryResponseParser getQueryResponseParser(final IndexQueryResponse<QueryResponse> queryRsp, final MetadataReference mdRef)

			throws IndexClientException {

		QueryResponse response = queryRsp.getContextualQueryResponse();

		return new SolrResponseParser(highlightUtils, browseAliases.get(mdRef), returnEmptyFields, includeRanking, response);

	}

}

package eu.dnetlib.functionality.index.query;

import java.util.Arrays;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.common.params.FacetParams;

import com.google.common.collect.BiMap;

import eu.dnetlib.functionality.index.client.AbstractIndexClient;

import eu.dnetlib.functionality.index.client.IndexClientException;

import eu.dnetlib.functionality.index.cql.TranslatedQuery;

import eu.dnetlib.functionality.index.query.Pruner.Result;

import eu.dnetlib.functionality.index.solr.utils.HighlightUtils;

import eu.dnetlib.functionality.index.utils.IndexFieldUtility;

import eu.dnetlib.functionality.index.utils.MetadataReference;

/**

 * A factory for creating SolrIndexQuery objects.

 */

public class SolrIndexQueryFactory extends IndexQueryFactory {

	/**

	 * logger.

	 */

	private static final Log log = LogFactory.getLog(SolrIndexQueryFactory.class);

	/** The Property name SERVICE_HIGHLIGHT_ENABLE. */

	private static final String SERVICE_HIGHLIGHT_ENABLE = "service.index.solr.highlight.enable";

	private static String GROUPBY = "&groupby=";

	private static String AND = " and ";

	private static String CLAUSE_PREFIX = "solr." + FacetParams.FACET_FIELD + "=";

	private static String BROWSE_PREFIX = "(>solr=SOLR solr." + FacetParams.FACET + "=true" + AND;

	private static String GROUP_BY_DELIMITER = ",";

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.functionality.index.query.IndexQueryFactory#newInstance(eu.dnetlib.functionality.index.cql.TranslatedQuery,

	 * eu.dnetlib.functionality.index.query.Pruner.Result, eu.dnetlib.functionality.index.query.QueryLanguage)

	 */

	@Override

	protected IndexQuery newInstance(final TranslatedQuery cql, final Result res, final QueryLanguage lang) {

		switch (lang) {

		case CQL:

			return new SolrIndexQuery(cql, res.getOptionMap());

		case SOLR:

			return new SolrIndexQuery(res.getNode().toCQL(), res.getOptionMap());

		default:

			throw new IllegalArgumentException("invalid query language: " + lang);

		}

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.functionality.index.query.IndexQueryFactory#setQueryOptions(eu.dnetlib.functionality.index.query.IndexQuery,

	 * eu.dnetlib.functionality.index.IndexServerDAO)

	 */

	@Override

	protected IndexQuery setQueryOptions(final IndexQuery indexQuery, final AbstractIndexClient client) {

		final SolrIndexQuery solrQuery = (SolrIndexQuery) indexQuery;

		boolean isHighlightEnabled = Boolean.parseBoolean(client.getServiceProperties().get(SERVICE_HIGHLIGHT_ENABLE));

		if (solrQuery.getHighlight() & isHighlightEnabled) {

			solrQuery.setHighlightFragsize(0).setHighlightSnippets(1).setHighlightSimplePre(HighlightUtils.DEFAULT_HL_PRE)

			.setHighlightSimplePost(HighlightUtils.DEFAULT_HL_POST).addHighlightField(IndexFieldUtility.RESULT)

			.addField(IndexFieldUtility.INDEX_RECORD_ID);

		}

		solrQuery.addField(IndexFieldUtility.RESULT);

		if (solrQuery.getFacetFields() != null) {

			log.debug("getFacetFields() " + Arrays.asList(solrQuery.getFacetFields()));

			solrQuery.setFacetMinCount(1);

		}

		return solrQuery;

	}

	@Override

	public String getBrowsingFields(String query, final MetadataReference mdRef) throws IndexClientException {

		if (query.contains(GROUPBY)) {

			final BiMap<String, String> aliases = browseAliases.get(mdRef);

			final String cql = query.split(GROUPBY)[0];

			String clauses = query.split(GROUPBY)[1].replaceAll(GROUP_BY_DELIMITER, AND);

			String newclauses = "";

			for (String clause : clauses.split(AND)) {

				final String cleanClause = clause.trim().toLowerCase();

				String clauseAlias = cleanClause;

				if ((aliases != null) && !aliases.isEmpty()) {

					clauseAlias = aliases.get(cleanClause) == null ? aliases.inverse().get(cleanClause) : aliases.get(cleanClause);

				}

				final String fieldName = clauseAlias != null ? clauseAlias : cleanClause;

				clauses = clauses.replaceAll(clause, CLAUSE_PREFIX + fieldName);

				if (!newclauses.isEmpty()) {

					newclauses += AND;

				}

				newclauses += CLAUSE_PREFIX + fieldName;

			}

			query = BROWSE_PREFIX + newclauses + ")" + AND + cql;

			log.debug("low level browse query " + query);

		}

		return query;

	}

}

package eu.dnetlib.functionality.index.query;

import static eu.dnetlib.miscutils.collections.MappedCollection.listMap;

import java.util.Collection;

import java.util.List;

import java.util.Map;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.response.FacetField;

import org.apache.solr.client.solrj.response.FacetField.Count;

import org.apache.solr.client.solrj.response.QueryResponse;

import org.apache.solr.common.SolrDocument;

import org.apache.solr.common.SolrDocumentList;

import com.google.common.base.Predicate;

import com.google.common.collect.BiMap;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import eu.dnetlib.data.provision.index.rmi.BrowsingRow;

import eu.dnetlib.data.provision.index.rmi.GroupResult;

import eu.dnetlib.functionality.index.utils.IndexFieldUtility;

import eu.dnetlib.miscutils.functional.UnaryFunction;

/**

 * The Class SolrResponseParser.

 */

public class SolrResponseParser extends QueryResponseParser {

	/**

	 * logger.

	 */

	private static final Log log = LogFactory.getLog(SolrResponseParser.class);

	/**

	 * Lower level response.

	 */

	private QueryResponse queryRsp = null;

	/** The wrapper rank. */

	protected final UnaryFunction<String, SolrDocument> wrapperRank = new UnaryFunction<String, SolrDocument>() {

		@Override

		public String evaluate(final SolrDocument doc) {

			return addRanking(getSingleField(doc, IndexFieldUtility.RESULT), getSingleField(doc, IndexFieldUtility.SCORE_FIELD));

		}

	};

	/** The wrapper no rank. */

	protected final UnaryFunction<String, SolrDocument> wrapperNoRank = new UnaryFunction<String, SolrDocument>() {

		@Override

		public String evaluate(final SolrDocument doc) {

			return wrap(getSingleField(doc, IndexFieldUtility.RESULT));

		}

	};

	/**

	 * The Constructor.

	 *

	 * @param highlightUtils

	 *            the highlight utils

	 * @param aliases

	 *            the aliases

	 * @param returnEmptyFields

	 *            the return empty fields

	 * @param includeRanking

	 *            the include ranking

	 * @param response

	 *            the response

	 */

	public SolrResponseParser(final UnaryFunction<String, String> highlightUtils, final BiMap<String, String> aliases, final boolean returnEmptyFields,

			final boolean includeRanking, final QueryResponse response) {

		super(highlightUtils, aliases, returnEmptyFields, includeRanking);

		this.queryRsp = response;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.query.QueryResponseParser#getNumFound()

	 */

	@Override

	public long getNumFound() {

		return this.queryRsp.getResults().getNumFound();

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.query.QueryResponseParser#getQueryTime()

	 */

	@Override

	public int getQueryTime() {

		return queryRsp.getQTime();

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.query.QueryResponseParser#getElapsedTime()

	 */

	@Override

	public long getElapsedTime() {

		return queryRsp.getElapsedTime();

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.query.QueryResponseParser#getStatus()

	 */

	@Override

	public String getStatus() {

		return String.valueOf(queryRsp.getStatus());

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.query.QueryResponseParser#getCurrentSize()

	 */

	@Override

	public int getCurrentSize() {

		return queryRsp.getResults().size();

	}

	/**

	 * Gets the query response.

	 *

	 * @return the query response

	 */

	public QueryResponse getQueryResponse() {

		return queryRsp;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.query.QueryResponseParser#getResults()

	 */

	@Override

	public List<String> getResults() {

		return asRankedList(queryRsp.getResults());

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.query.QueryResponseParser#getNumberOfBrowsingResults()

	 */

	@Override

	public Long getNumberOfBrowsingResults() {

		List<FacetField> ffList = queryRsp.getFacetFields();

		Long maxCount = 0L;

		if (ffList != null) {

			for (FacetField ff : ffList) {

				if (ff != null) {

					Long countFacets = countFacets(ff.getValues());

					if (countFacets > maxCount) {

						maxCount = countFacets;

					}

				}

			}

		}

		return maxCount;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.query.QueryResponseParser#getBrowsingResults()

	 */

	@Override

	public List<BrowsingRow> getBrowsingResults() {

		List<BrowsingRow> bresList = Lists.newArrayList();

		List<GroupResult> facets = Lists.newArrayList();

		final List<FacetField> ffList = queryRsp.getFacetFields();

		Long numberOfBrowsingResults = getNumberOfBrowsingResults();

		for (int i = 0; (ffList != null) && (i < numberOfBrowsingResults); i++) {

			for (FacetField ff : ffList) {

				String name = aliases.inverse().get(ff.getName());

				// fix #1456

				if (name == null) {

					name = ff.getName();

				}

				final Count facet = getFacet(ff, i);

				if ((facet != null) && (facet.getCount() > 0)) {

					final String value = facet.getName();

					final int count = (int) facet.getCount();

					if (returnEmptyFields || !value.isEmpty()) {

						facets.add(new GroupResult(name, value, count));

					}

				}

			}

			if (facets.size() > 0) {

				bresList.add(new BrowsingRow(Lists.newArrayList(facets)));

				facets.clear();

			}

		}

		if (log.isDebugEnabled()) {

			log.debug("BrowsingResult size: " + bresList.size());

		}

		return bresList;

	}

	// /////////////// helpers

	/**

	 * Gets the facet.

	 *

	 * @param ff

	 *            the ff

	 * @param pos

	 *            the pos

	 * @return the facet

	 */

	private Count getFacet(final FacetField ff, final int pos) {

		if ((ff.getValues() == null) || (pos >= ff.getValues().size())) return null;

		return ff.getValues().get(pos);

	}

	/**

	 * Given SolrDocumentList, return a List of Strings, representing it.

	 *

	 * @param documentList

	 *            the document list

	 * @return the list< string>

	 */

	private List<String> asRankedList(final SolrDocumentList documentList) {

		UnaryFunction<String, SolrDocument> wrapper = includeRanking ? wrapperRank : wrapperNoRank;

		if (queryRsp.getHighlighting() != null) return listMap(listMap(documentList, new UnaryFunction<String, SolrDocument>() {

			@Override

			public String evaluate(final SolrDocument doc) {

				String score = getSingleField(doc, IndexFieldUtility.SCORE_FIELD);

				String hl = getHighlighting(getSingleField(doc, IndexFieldUtility.INDEX_RECORD_ID));

				String res = hl != null ? hl : getSingleField(doc, IndexFieldUtility.RESULT);

				return includeRanking ? addRanking(res, score) : wrap(res);

			}

		}), highlightUtils);

		return listMap(documentList, wrapper);

	}

	/**

	 * Converts a String document to

	 *

	 * <record rank="score"> [document] </record>.

	 *

	 * @param doc

	 *            the doc

	 * @param score

	 *            the score

	 * @return the string

	 */

	private String addRanking(final String doc, final String score) {

		return new String("<record rank=\"" + score + "\">" + doc + "</record>");

	}

	/**

	 * Wraps the given document as <record> [document] </record>.

	 *

	 * @param doc

	 *            the doc

	 * @return the string

	 */

	private String wrap(final String doc) {

		return new String("<record>" + doc + "</record>");

	}

	/**

	 * Gets the single field.

	 *

	 * @param doc

	 *            the doc

	 * @param fieldName

	 *            the field name

	 * @return the single field

	 */

	@SuppressWarnings("unchecked")

	protected String getSingleField(final SolrDocument doc, final String fieldName) {

		Object value = doc.getFieldValue(fieldName);

		if (value instanceof Collection) return Iterables.getOnlyElement((Iterable<String>) value);

		return String.valueOf(value);

	}

	/**

	 * Gets the highlighting.

	 *

	 * @param docId

	 *            the doc id

	 * @return the highlighting

	 */

	private String getHighlighting(final String docId) {

		final Map<String, List<String>> highlight = queryRsp.getHighlighting().get(docId);

		String result = new String();

		if ((highlight != null) && (highlight.get(IndexFieldUtility.RESULT) != null)) {

			for (String s : highlight.get(IndexFieldUtility.RESULT)) {

				result = result.concat(s);

			}

			return result;

		}

		return null;

	}

	/**

	 * helper method.

	 *

	 * @param facets

	 *            the list of facets to analyze

	 * @return the number of non-empty facets in the list whose count is greater than zero

	 */

	private Long countFacets(final List<Count> facets) {

		if (facets == null) return 0L;

		return (long) Iterables.size(Iterables.filter(facets, new Predicate<Count>() {

			@Override

			public boolean apply(final Count c) {

				return (c != null) && (c.getName() != null) && !c.getName().isEmpty() && (c.getCount() > 0);

			}

		}));

	}

	@Override

	public long getStart() {

		// TODO Auto-generated method stub

		return queryRsp.getResults().getStart();

	}

}

package eu.dnetlib.functionality.index.query;

import org.apache.solr.client.solrj.response.QueryResponse;

/**

 * The Class SolrIndexQueryResponse.

 */

public class SolrIndexQueryResponse implements IndexQueryResponse<QueryResponse> {

	private QueryResponse solrQueryResponse;

	public SolrIndexQueryResponse(final QueryResponse solrQueryResponse) {

		this.solrQueryResponse = solrQueryResponse;

	}

	@Override

	public QueryResponse getContextualQueryResponse() {

		return solrQueryResponse;

	}

}

package eu.dnetlib.functionality.index.query;

import java.util.List;

import java.util.Map;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.SolrQuery;

import org.apache.solr.common.params.ModifiableSolrParams;

import org.apache.solr.common.params.SolrParams;

import eu.dnetlib.functionality.index.cql.QueryOptions;

import eu.dnetlib.functionality.index.cql.TranslatedQuery;

import eu.dnetlib.functionality.index.utils.IndexFieldUtility;

/**

 * The Class SolrIndexQuery.

 * 

 * @author claudio, sandro

 */

public class SolrIndexQuery extends SolrQuery implements IndexQuery {

	/** The Constant serialVersionUID. */

	private static final long serialVersionUID = 1L;

	/**

	 * logger.

	 */

	private static final Log log = LogFactory.getLog(SolrIndexQuery.class);

	/**

	 * Instantiates a new solr index query.

	 * 

	 * @param query

	 *            the query

	 * @param options

	 *            the options

	 */

	public SolrIndexQuery(final TranslatedQuery query, final Map<String, List<String>> options) {

		this(query.asLucene(), options);

		setCqlParams(query.getOptions());

		log.debug("internal solr query: " + this.toString());

	}

	/**

	 * Instantiates a new solr index query.

	 * 

	 * @param query

	 *            the query

	 * @param options

	 *            the options

	 */

	public SolrIndexQuery(final String query, final Map<String, List<String>> options) {

		this(query);

		// TODO verify that the input options belongs to solr

		super.add(getQueryParams(options));

	}

	/**

	 * Instantiates a new solr index query.

	 * 

	 * @param query

	 *            the query

	 */

	public SolrIndexQuery(final String query) {

		super(query);

	}

	@Override

	public IndexQuery setQueryOffset(final int offset) {

		super.setStart(offset);

		return this;

	}

	@Override

	public IndexQuery setQueryLimit(final int limit) {

		super.setRows(limit);

		return this;

	}