D-Net

import java.util.Map;

import java.util.function.Function;

import org.springframework.beans.factory.annotation.Required;

/**

 * Factory for the SolrTypeBasedCqlValueTransformerMap class objects

 * @author claudio

 */

public class SolrTypeBasedCqlValueTransformerMapFactory {

	/**

	 * Method returns a new instance of SolrTypeBasedCqlValueTransformerMap.

	 * @param schema

	 */

	public SolrTypeBasedCqlValueTransformerMap getIt(final Map<String, ValueType> schema) {

		return new SolrTypeBasedCqlValueTransformerMap(schema, getTransformerMap());

	}

	@Required

	public void setTransformerMap(Map<String, Function<String, String>> transformerMap) {

		this.transformerMap = transformerMap;

	}

}

/**

 * Simply and not very roboust normalizer for solr dates. Basically it handles well yyyy-mm-dd and

 * yyyy-mm-ddThh:mm:ssZ

 * @author marko

 */

public class SimpleDateValueTransformer implements Function<String, String> {

import java.util.Map;

import java.util.function.Function;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.common.SolrException;

/**

 * This class maps the fields in the given index schema with a transformation rule.

 * @author marko

 */

public class SolrTypeBasedCqlValueTransformerMap implements CqlValueTransformerMap {

	/**

	 * Create value transformer map bound to a specific schema

	 * @param schema

	 */

	public SolrTypeBasedCqlValueTransformerMap(final Map<String, ValueType> schema, final Map<String, Function<String, String>> transformerMap) {

		this.schema = schema;

	/**

	 * {@inheritDoc}

	 */

	@Override

	public Function<String, String> transformerFor(final String fieldName) {

		} catch (SolrException e) {

			log.debug("cannot find field", e);

		}

	}

import java.util.HashMap;

import java.util.Map;

import org.apache.solr.common.SolrInputDocument;

import org.apache.solr.common.SolrInputField;

// TODO: Auto-generated Javadoc

/**

	/**

	 * Instantiates a new solr index document.

	 *

	 */

	public SolrIndexDocument(final Map<String, ValueType> schema, final String dsId) {

		super(schema, dsId);

	/**

	 * Instantiates a new solr index document.

	 *

	 */

	public SolrIndexDocument(final Map<String, ValueType> schema, final String dsId, final SolrInputDocument solrDocument) {

		super(schema, dsId);

	/**

	 * Adds the fields.

	 *

	 */

	private void addFields(final SolrInputDocument solrDocument) {

		for (String name : solrDocument.getFieldNames()) {

			Collection<Object> fieldValues = solrDocument.getFieldValues(name);

				addField(name, fieldValues);

				addField(name, fieldValues.iterator().next());

		}

	}

	/**

	 * Sets the content.

	 *

	 */

	public void setContent(final SolrInputDocument solrDocument) {

		addFields(solrDocument);

	 */

	public SolrInputDocument getSolrDocument() {

		for (String key : fields.keySet()) {

			SolrInputField solrField = new SolrInputField(key);

			for (Object o : fields.get(key)) {

			}

			data.put(key, solrField);

		}

import java.io.IOException;

import java.io.StringReader;

import java.util.Map;

import eu.dnetlib.clients.index.model.Any.ValueType;

import eu.dnetlib.clients.index.query.IndexQueryFactory;

import eu.dnetlib.index.AbstractBackendDescriptor;

import eu.dnetlib.index.IndexCollection;

import eu.dnetlib.index.IndexServerDAO;

import eu.dnetlib.index.feed.DocumentMapperFactory;

import eu.dnetlib.index.query.SolrIndexQueryFactory;

import eu.dnetlib.index.query.SolrIndexQueryResponseFactory;

import eu.dnetlib.index.solr.feed.SolrDocumentMapperFactory;

import eu.dnetlib.index.utils.IndexConfigFactory;

import eu.dnetlib.index.utils.RemoteSolrAdministrator;

import eu.dnetlib.index.utils.ZkUtils;

import eu.dnetlib.rmi.provision.IndexServiceException;

/**

 * The Class SolrIndexServerDAO.

	 */

	private static final Log log = LogFactory.getLog(SolrIndexServerDAO.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Autowired

	private ZkUtils zkUtils;

	@Autowired

	private SolrIndexQueryResponseFactory queryResponseFactory;

	@Autowired

	private SolrIndexQueryFactory solrIndexQueryFactory;

	@Autowired

	private SolrDocumentMapperFactory solrDocumentMapperFactory;

	private RemoteSolrAdministrator solrAdministrator;

	@Autowired

	private SolrTypeBasedCqlValueTransformerMapFactory tMapFactory;

	/**

	 * {@inheritDoc}

	 */

	@Override

	public void createIndexCollection(final MetadataReference mdref, final String fields) throws IndexServiceException {

			client.connect();

				params.put("numShards", p.get("numShards"));

				params.put("replicationFactor", p.get("replicationFactor"));

				for (IndexConfigFactory.CONFIG_PARAMS param_Name : IndexConfigFactory.CONFIG_PARAMS.values()) {

					params.put(param_Name.toString(), p.get(param_Name.toString()));

				}

				solrAdministrator.createSolrIndex(p.get("host"), p.get("port"), mdref.toString(), p.get("numShards"), p.get("replicationFactor"),

			}

			client.getZkStateReader().close();

		} catch (Exception e) {

			log.error("Error on creating IndexCollection", e);

			throw new IndexServiceException("Error on creating IndexCollection", e);

		}

	}

	@Override

	public void updateIndexCollection(final MetadataReference mdRef, final Document fields) throws IndexServiceException {

			client.connect();

			params.put("numShards", getServiceProperties().get("numShards"));

			params.put("replicationFactor", getServiceProperties().get("replicationFactor"));

			for (IndexConfigFactory.CONFIG_PARAMS param_Name : IndexConfigFactory.CONFIG_PARAMS.values()) {

				params.put(param_Name.toString(), getServiceProperties().get(param_Name.toString()));

			}

			zkUtils.uploadZookeperConfig(client.getZkStateReader().getZkClient(), mdRef.toString(), fields, params, true);

			client.getZkStateReader().close();

			solrAdministrator.reloadCollection(getServiceProperties().get("host"), getServiceProperties().get("port"), mdRef.toString());

		} catch (Exception e) {

			log.error("Error on updating IndexCollection", e);

			throw new IndexServiceException("Error on updating IndexCollection", e);

		}

	}

	/**

	 * Parses the fields parameter.

	 * @return the document

	 */

	private Document parse(final String fields) throws IndexServiceException {

		try {

	/**

	 * {@inheritDoc}

	 */

	@Override

	public IndexCollection getIndexCollection(final MetadataReference mdref) throws IndexServiceException {

	}

	/**

	 * {@inheritDoc}

	 */

	@Override

	public Map<String, ValueType> getSchema(final MetadataReference mdRef) throws IndexServiceException {

		return fields;

	}

	/**

	 * {@inheritDoc}

	 */

	@Override

	public CqlValueTransformerMap getCqlValueTransformerMap(final MetadataReference mdRef) throws IndexServiceException {

	/**

	 * {@inheritDoc}

	 */

	@Override

	public DocumentMapperFactory getDocumentMapperFactory() throws IndexServiceException {

	/**

	 * {@inheritDoc}

	 */

	@Override

	public void shutdown(final MetadataReference mdRef) throws IndexServiceException {

		try {

			getClient(mdRef).close();

		} catch (IOException e) {

		}

	}

	/**

	 * Gets a server with the default collection set according to the given mdRef.

	 * @return a server instance

	 */

	private CloudSolrClient getClient(final MetadataReference mdRef) throws IndexServiceException {

		CloudSolrClient client = getClient();

	/**

	 * Gets the server.

	 * @return a server instance

	 */

	private CloudSolrClient getClient() {

		String address = getEndpoint().get(ADDRESS);

		log.info("connecting to address: " + address);

	}

	/**

	 * Gets the solr administrator.

	 * @return the solrAdministrator

	 */

	public RemoteSolrAdministrator getSolrAdministrator() {

	/**

	 * Sets the solr administrator.

	 */

	@Required

	public void setSolrAdministrator(final RemoteSolrAdministrator solrAdministrator) {

import java.io.IOException;

import java.util.Collection;

import java.util.Iterator;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.response.UpdateResponse;

import org.apache.solr.common.SolrInputDocument;

/**

 * The Class SolrIndexCollection.

 */

public class SolrIndexCollection implements IndexCollection {

	/**

	 * The log.

	 */

	private static final Log log = LogFactory.getLog(SolrIndexCollection.class); // NOPMD by marko on 11/24/08 5:02 PM

	private CloudSolrClient client;

	private boolean shutdown = false;

	/**

	 * The Constructor.

	 *

	 */

		client.connect();

	}

	/**

	 * {@inheritDoc}

	 */

	@Override

	public boolean add(final IndexDocument doc) throws IndexServiceException {

		if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");

		final SolrIndexDocument solrDocument = (SolrIndexDocument) doc;

		try {

			final UpdateResponse response = client.add(solrDocument.getSolrDocument());

	/**

	 * {@inheritDoc}

	 */

	@Override

	public boolean addAll(final Iterator<IndexDocument> docs) throws IndexServiceException {

		if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");

		try {

			return response.getStatus() == 0;

		} catch (final Exception e) {

			throw new IndexServiceException("Unable to add document", e);

	/**

	 * {@inheritDoc}

	 */

	@Override

	public boolean addAll(final Collection<IndexDocument> docs) throws IndexServiceException {

	/**

	 * {@inheritDoc}

	 */

	@Override

	public boolean deleteIndex(final String dsId) throws IndexServiceException {

	/**

	 * {@inheritDoc}

	 */

	@Override

	public boolean deleteByQuery(final String query, final String dsId) throws IndexServiceException {

	/**

	 * Do delete.

	 *

	 * @return true, if do delete

	 */

	protected boolean doDelete(final String query) throws IndexServiceException {

		if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");

	/**

	 * {@inheritDoc}

	 */

	@Override

	public boolean commit() throws IndexServiceException {

		try {

			client.close();

		} catch (IOException e) {

		}

		shutdown = true;

	}

import java.io.StringReader;

import java.io.StringWriter;

import java.util.Iterator;

import java.util.List;

import javax.xml.stream.events.Namespace;

import javax.xml.stream.events.StartElement;

import javax.xml.stream.events.XMLEvent;

import org.apache.solr.common.SolrInputDocument;

/**

 * Optimized version of the document parser, drop in replacement of InputDocumentFactory.

 * <p>

 * Faster because:

 * </p>

 * <ul>

 * <li>Quickly serialize the 'result' element directly in a string.</li>

 * <li>Uses less memory: less pressure on GC and allows more threads to process this in parallel</li>

 * </ul>

 * <p>

 * This class is fully reentrant and can be invoked in parallel.

 * </p>

 *

 * @author marko

 */

public class StreamingInputDocumentFactory extends InputDocumentFactory {

	protected static final String DEFAULTDNETRESULT = "dnetResult";

	protected static final String ROOT_ELEMENT = "indexRecord";

	protected ThreadLocal<XMLInputFactory> inputFactory = new ThreadLocal<XMLInputFactory>() {

		@Override

		}

	};

	@Override

	public SolrInputDocument parseDocument(final String version, final String inputDocument, final String dsId, final String resultName)

			throws XMLStreamException {

		return parseDocument(version, inputDocument, dsId, resultName, null);

	}

	@Override

	public SolrInputDocument parseDocument(final String version,

			final String inputDocument,

			final ResultTransformer resultTransformer) {

		final StringWriter results = new StringWriter();

		try {

			XMLEventReader parser = inputFactory.get().createXMLEventReader(new StringReader(inputDocument));

			while (parser.hasNext()) {

				final XMLEvent event = parser.nextEvent();

	}

	private List<Namespace> getNamespaces(final XMLEvent event) {

		@SuppressWarnings("unchecked")

		Iterator<Namespace> nsIter = event.asStartElement().getNamespaces();

		while (nsIter.hasNext()) {

	 * Helper used to get the string from a text element.

	 *

	 * @param text

	 */

	protected final String getText(final XMLEvent text) {

		if (text.isEndElement()) // log.warn("skipping because isEndOfElement " + text.asEndElement().getName().getLocalPart());

			return "";

	}

}

package eu.dnetlib.index.solr.feed;

/**

 * Created by claudio on 17/11/15.

 */

public abstract class ResultTransformer implements Function<String, String> {

	protected Mode mode;

	public ResultTransformer(final Mode mode) {

		this.mode = mode;

	}

}

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.Arrays;

import java.util.List;

import javax.xml.stream.XMLStreamException;

import org.dom4j.DocumentException;

/**

 * @author claudio

 */

public abstract class InputDocumentFactory {

	private final static List<String> dateFormats = Arrays.asList("yyyy-MM-dd'T'hh:mm:ss", "yyyy-MM-dd", "dd-MM-yyyy", "dd/MM/yyyy", "yyyy");

	/**

	 * method return a solr-compatible string representation of a date

	 *

	 * @param date

	 * @throws DocumentException

	 * @throws ParseException

	 */

	public static String getParsedDateField(final String date) {

		for (String formatString : dateFormats) {

			try {

			} catch (ParseException e) {}

		}

		throw new IllegalStateException("unable to parse date: " + date);

	}

	public String parseDate(final String date) {

		return getParsedDateField(date);

	}

import java.util.Map;

import java.util.function.Function;

import javax.xml.stream.XMLStreamException;

import eu.dnetlib.clients.index.model.Any.ValueType;

import eu.dnetlib.clients.index.model.document.IndexDocument;

import eu.dnetlib.clients.index.model.document.Status;

import eu.dnetlib.index.feed.DocumentMapperFactory;

import eu.dnetlib.index.solr.model.SolrIndexDocument;

/**

 * A factory for creating SolrDocumentMapper objects.

 */

public class SolrDocumentMapperFactory implements DocumentMapperFactory {

}

package eu.dnetlib.index.solr.browsing;

import java.util.List;

import javax.xml.bind.annotation.XmlAccessType;

import javax.xml.bind.annotation.XmlAccessorType;

import javax.xml.bind.annotation.XmlElement;

import org.springframework.beans.factory.annotation.Required;

/**

 * serialization of the browsing result.

 * <row> <groupresult field="facetFieldName1"> <value>facetFieldValue</value> <count>1</count> </groupresult>

 * <groupresult field="facetFieldName2"> <value>facetFieldValue</value> <count>1</count> </groupresult>

 * </row>

 *

 * @author claudio

 */

@XmlRootElement(namespace = "", name = "row")

@XmlAccessorType(XmlAccessType.FIELD)

	@XmlElement(name = "groupresult", required = true)

	private List<GroupResult> groupresult;

	public BrowsingRow(final List<GroupResult> groupresult) {

		this.groupresult = groupresult;

 * </pre>

 *

 * @author claudio

 */

@XmlRootElement(namespace = "", name = "groupresult")

public class GroupResult {

	private int count;

	/**

	 * Builds a groupResult.

	 *

	 * @param count

	 */

	public GroupResult(final String name, final String value, final int count) {

		return name;

	}

	@Required

	public void setName(final String name) {

		this.name = name;

	}

	@Required

	public void setValue(final String value) {

		this.value = value;

	}

	@Required

	public void setCount(final int count) {

		this.count = count;

package eu.dnetlib.index.solr.client;

import java.io.IOException;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import com.google.common.collect.BiMap;

import com.google.common.collect.Maps;

import eu.dnetlib.clients.index.client.IndexClient;

import eu.dnetlib.clients.index.client.IndexClientException;

import eu.dnetlib.clients.index.client.response.BrowseEntry;

import eu.dnetlib.clients.index.query.QueryLanguage;

import eu.dnetlib.clients.index.query.QueryResponseFactory;

import eu.dnetlib.clients.index.query.QueryResponseParser;

import eu.dnetlib.cql.CqlValueTransformerMap;

import eu.dnetlib.index.query.SolrIndexQuery;

import eu.dnetlib.index.query.SolrIndexQueryFactory;

import eu.dnetlib.index.query.SolrIndexQueryResponse;

import eu.dnetlib.index.solr.cql.SolrTypeBasedCqlValueTransformerMapFactory;

import eu.dnetlib.rmi.provision.BrowsingRow;

import eu.dnetlib.rmi.provision.GroupResult;

/**

 * The Class SolrIndexClient.

 */

	private static final Log log = LogFactory.getLog(SolrIndexClient.class);

	private CloudSolrClient client;

	private SolrIndexQueryFactory solrIndexQueryFactory;

	private QueryResponseFactory<QueryResponse> queryResponseFactory;

	private SolrTypeBasedCqlValueTransformerMapFactory tMapFactory;

	/**

	 * The Constructor.

	 *

	 * @param tMapFactory

	 */

	public SolrIndexClient(final String format, final String layout, final String interpretation, final Map<String, String> serviceProperties,

			final SolrIndexQueryFactory indexQueryFactory, final QueryResponseFactory<QueryResponse> queryResponseFactory,

			final SolrTypeBasedCqlValueTransformerMapFactory tMapFactory) {

		this.solrIndexQueryFactory = indexQueryFactory;

		this.queryResponseFactory = queryResponseFactory;

		this.tMapFactory = tMapFactory;

	}

	/**

	 * Do delete.

	 *

	 * @return true, if do delete

	 */

	@Override

	public long delete(final String query) throws IndexClientException {

			String tquery = translatedQuery.getQuery();

			translatedQuery.setQueryLimit(0);

			QueryResponseParser responseParser = queryResponseFactory.getQueryResponseParser(rsp, mdRef);

			long total = responseParser.getNumFound();

			return total;

		} catch (Exception e) {

			throw new IndexClientException("unable to run delete by query: " + query, e);

	 * {@inheritDoc}

	 *

	 * @throws IndexClientException

	 */

	@Override

	public List<BrowseEntry> browse(final String query, final List<String> browseFields, final int max) throws IndexClientException {

	 * {@inheritDoc}

	 *

	 * @throws IndexClientException

	 */

	@Override

	public List<BrowseEntry> browse(final String query, final List<String> browseFields, final int max, final List<String> filterQuery)

	}

			throws IndexClientException {

		log.debug("Browse request for the index collection for query:" + query);

			final MetadataReference mdRef,

			final List<String> browseFields) throws IndexClientException {

		try {

			QueryResponseParser responseParser = queryResponseFactory.getQueryResponseParser(response, mdRef);

			List<BrowsingRow> results = responseParser.getBrowsingResults();

			List<BrowseEntry> out = convertBrowseEntry(browseFields, results, responseParser.getAliases());

			return out;

			throw new IndexClientException("Error on executing a query " + originalQuery, e);

		}

	}

	 * Creates the connection.

	 *

	 * @return the string

	 */

	private String getUrl() throws IndexClientException {

		String address = serviceProperties.get(ZK_ADDRESS);

		return address;

	}

	/**

	 *

	 */

		if (this.client == null) {

			String url = getUrl();

			log.debug("create new Client " + url);

			client.setDefaultCollection(String.format("%s-%s-%s", getFormat(), getLayout(), getInterpretation()));

			try {

				client.ping();

	}

	/**

	 *