D-Net

{ "type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-index-solr-service/branches/solr75", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-index-solr-service_solr75" }

package eu.dnetlib.functionality.index.solr.feed;

import com.google.common.base.Function;

import eu.dnetlib.functionality.index.feed.DocumentMapperFactory;

import eu.dnetlib.functionality.index.model.Any.ValueType;

import eu.dnetlib.functionality.index.model.document.IndexDocument;

import eu.dnetlib.functionality.index.model.document.Status;

import eu.dnetlib.functionality.index.query.SolrIndexDocument;

import eu.dnetlib.functionality.index.utils.MetadataReference;

import org.springframework.beans.factory.annotation.Required;

import javax.xml.stream.XMLStreamException;

import java.util.Map;

/**

 * A factory for creating SolrDocumentMapper objects.

 */

public class SolrDocumentMapperFactory implements DocumentMapperFactory {

    protected static final String DNETRESULT = "result";

    /**

     * document factory used for the feed process.

     */

    private InputDocumentFactory documentFactory;

    private Function<String, IndexDocument> getRecordMapper(final Map<String, ValueType> schema,

                                                            final String dsId,

                                                            final String version) {

        return doc -> {

            SolrIndexDocument indexDocument = new SolrIndexDocument(schema, dsId);

            try {

                indexDocument.setContent(documentFactory.parseDocument(version, doc, dsId, DNETRESULT));

            } catch (XMLStreamException e) {

                return indexDocument.setMarked();

            }

            indexDocument.setStatus(Status.OK);

            return indexDocument;

        };

    }

    @Override

    public Function<String, IndexDocument> getRecordMapper(final Map<String, ValueType> schema, final MetadataReference mdRef, final String dsId, final String version, final boolean emptyResult) {

        if (emptyResult == false) return getRecordMapper(schema, dsId, version);

        else {

            return doc -> {

                SolrIndexDocument indexDocument = new SolrIndexDocument(schema, dsId);

                try {

                    ResultTransformer transformer = new ResultTransformer(ResultTransformer.Mode.empty) {

                        @Override

                        public String apply(String input) {

                            return input;

                        }

                    };

                    indexDocument.setContent(documentFactory.parseDocument(version, doc, dsId, DNETRESULT, transformer));

                } catch (XMLStreamException e) {

                    return indexDocument.setMarked();

                }

                indexDocument.setStatus(Status.OK);

                return indexDocument;

            };

        }

    }

    /**

     * Gets the document factory.

     *

     * @return the documentFactory

     */

    public InputDocumentFactory getDocumentFactory() {

        return documentFactory;

    }

    /**

     * Sets the document factory.

     *

     * @param documentFactory the documentFactory to set

     */

    @Required

    public void setDocumentFactory(final InputDocumentFactory documentFactory) {

        this.documentFactory = documentFactory;

    }

}

package eu.dnetlib.functionality.index.solr.browsing;

import java.util.List;

import javax.xml.bind.annotation.XmlAccessType;

import javax.xml.bind.annotation.XmlAccessorType;

import javax.xml.bind.annotation.XmlElement;

import javax.xml.bind.annotation.XmlRootElement;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.data.provision.index.rmi.GroupResult;

/**

 *

 * serialization of the browsing result.

 *

 * <row> <groupresult field="facetFieldName1"> <value>facetFieldValue</value> <count>1</count> </groupresult>

 * 

 * <groupresult field="facetFieldName2"> <value>facetFieldValue</value> <count>1</count> </groupresult>

 * 

 * </row>

 *

 * @author claudio

 *

 */

@XmlRootElement(namespace = "", name = "row")

@XmlAccessorType(XmlAccessType.FIELD)

public class BrowsingRow {

	@XmlElement(name = "groupresult", required = true)

	private List<GroupResult> groupresult;

	public BrowsingRow() {}

	public BrowsingRow(final List<GroupResult> groupresult) {

		this.groupresult = groupresult;

	}

	/**

	 * adds a GroupResult.

	 *

	 * @param fieldName

	 * @param fieldValue

	 * @param count

	 */

	public void addBrowsingRow(final String fieldName, final String fieldValue, final int count) {

		groupresult.add(new GroupResult(fieldName, fieldValue, count));

	}

	@Override

	public boolean equals(final Object obj) {

		if (!(obj instanceof BrowsingRow)) return false;

		final BrowsingRow brws = (BrowsingRow) obj;

		return groupresult.equals(brws.getGroupResult());

	}

	public List<GroupResult> getGroupResult() {

		return groupresult;

	}

	@Required

	public void setGroupResult(final List<GroupResult> groupresult) {

		this.groupresult = groupresult;

	}

}

package eu.dnetlib.functionality.index.solr.browsing;

import javax.xml.bind.annotation.XmlAttribute;

import javax.xml.bind.annotation.XmlRootElement;

import org.springframework.beans.factory.annotation.Required;

/**

 * <pre>

 * {@code

 * <groupresult field="facetFieldName">

 * 	<value>facetFieldValue</value>

 * 	<count>1</count>

 * </groupresult>

 * }

 * </pre>

 *

 * @author claudio

 *

 */

@XmlRootElement(namespace = "", name = "groupresult")

public class GroupResult {

	private String name;

	private String value;

	private int count;

	public GroupResult() {}

	/**

	 * Builds a groupResult.

	 *

	 * @param fieldName

	 * @param fieldValue

	 * @param count

	 */

	public GroupResult(final String name, final String value, final int count) {

		this.name = name;

		this.value = value;

		this.count = count;

	}

	@Override

	public boolean equals(final Object obj) {

		if (!(obj instanceof GroupResult)) return false;

		final GroupResult g = (GroupResult) obj;

		if ((this.getCount() == g.getCount()) && this.getName().equals(g.getName()) && this.getValue().equals(g.getValue())) return true;

		return false;

	}

	@XmlAttribute(name = "field", required = true)

	public String getName() {

		return name;

	}

	public String getValue() {

		return value;

	}

	public int getCount() {

		return count;

	}

	@Required

	public void setName(final String name) {

		this.name = name;

	}

	@Required

	public void setValue(final String value) {

		this.value = value;

	}

	@Required

	public void setCount(final int count) {

		this.count = count;

	}

}

package eu.dnetlib.functionality.index.utils;

import java.io.IOException;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerConfigurationException;

import javax.xml.transform.TransformerException;

import javax.xml.transform.TransformerFactory;

import javax.xml.transform.TransformerFactoryConfigurationError;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.io.DocumentResult;

import org.dom4j.io.DocumentSource;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Required;

import org.springframework.core.io.Resource;

import eu.dnetlib.data.provision.index.rmi.IndexServiceException;

public class IndexSchemaFactory {

	private static final Log log = LogFactory.getLog(IndexSchemaFactory.class); // NOPMD by marko on 11/24/08 5:02 PM

	private final static String TEXT_FIELD_TYPE = "textFieldType";

	private Resource schemaTemplate;

	private String textFieldType;

	/**

	 * Transformer used to get the index schema.

	 */

	private Transformer transformer;

	public void init() throws TransformerConfigurationException, TransformerFactoryConfigurationError, DocumentException, IOException {

		transformer = TransformerFactory.newInstance().newTransformer(new DocumentSource(new SAXReader().read(getSchemaTemplate().getInputStream())));

		transformer.setParameter(TEXT_FIELD_TYPE, getTextFieldType());

	}

	/**

	 * @param fields

	 *            input document

	 * @return the application of the schemaXslt transformation to the docIn document

	 * @throws IndexServiceException

	 */

	public String getSchema(final Document fields) throws IndexServiceException {

		log.debug("default field type: " + getTextFieldType());

		log.debug("building schema from fields: \n" + fields.asXML() + "\n");

		final DocumentResult result = new DocumentResult();

		try {

			transformer.transform(new DocumentSource(fields), result);

			String xml = result.getDocument().asXML();

			log.debug("new index schema:\n" + xml);

			return xml;

		} catch (TransformerException e) {

			throw new IndexServiceException(e);

		}

	}

	@Required

	public void setSchemaTemplate(final Resource schemaTemplate) {

		this.schemaTemplate = schemaTemplate;

	}

	public Resource getSchemaTemplate() {

		return schemaTemplate;

	}

	@Required

	public void setTextFieldType(final String textFieldType) {

		this.textFieldType = textFieldType;

	}

	public String getTextFieldType() {

		return textFieldType;

	}

}

package eu.dnetlib.functionality.index.utils;

import java.util.Map;

import java.util.Map.Entry;

import org.antlr.stringtemplate.StringTemplate;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import com.google.common.base.Predicate;

import com.google.common.collect.Maps;

import eu.dnetlib.data.provision.index.rmi.IndexServiceException;

public class IndexConfigFactory {

	private static final Log log = LogFactory.getLog(IndexConfigFactory.class); // NOPMD

																				// by

																				// marko

																				// on

																				// 11/24/08

																				// 5:02

																				// PM

	public enum CONFIG_PARAMS {

		// TODO replace with actual values

		luceneMatchVersion,

		serverLibPath,

		filterCacheSize,

		filterCacheInitialSize,

		queryCacheSize,

		queryCacheInitialSize,

		documentCacheSize,

		documentCacheInitialSize,

		queryResultWindowSize,

		queryResultMaxDocCached,

		ramBufferSizeMB,

		mergeFactor,

		termIndexInterval,

		autosoftcommit,

		autocommit,

		maxIndexingThreads;

		static boolean isValid(final String v) {

			try {

				CONFIG_PARAMS.valueOf(v);

				return true;

			} catch (Throwable e) {

				return false;

			}

		}

	};

	/**

	 * Index configuration template.

	 */

	private StringTemplate solrConfig;

	/**

	 * @param params

	 *            input document

	 * @return the application of the schemaXslt transformation to the docIn

	 *         document

	 * @throws IndexServiceException

	 */

	public String getConfig(final Map<String, String> params) throws IndexServiceException {

		final StringTemplate conf = new StringTemplate(getSolrConfig()

				.getTemplate());

		for (Entry<String, String> e : filter(params).entrySet()) {

			log.debug("setting conf property [" + e.getKey() + ":"

					+ e.getValue() + "]");

			conf.setAttribute(e.getKey(), e.getValue());

		}

		return conf.toString();

	}

	private Map<String, String> filter(final Map<String, String> params) {

		return Maps.filterKeys(params, input -> CONFIG_PARAMS.isValid(input));

	}

	public StringTemplate getSolrConfig() {

		return solrConfig;

	}

	@Required

	public void setSolrConfig(final StringTemplate solrConfig) {

		this.solrConfig = solrConfig;

	}

}

package eu.dnetlib.functionality.index.utils;

import java.io.IOException;

import java.io.InputStream;

import java.util.Map;

import java.util.Map.Entry;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.common.cloud.SolrZkClient;

import org.apache.zookeeper.CreateMode;

import org.apache.zookeeper.KeeperException;

import org.dom4j.Document;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

import org.springframework.core.io.Resource;

import org.springframework.core.io.support.PathMatchingResourcePatternResolver;

import com.google.common.collect.Maps;

import eu.dnetlib.data.provision.index.rmi.IndexServiceException;

/**

 * The Class ZkUtils.

 */

public class ZkUtils {

	/**

	 * The log.

	 */

	private static final Log log = LogFactory.getLog(ZkUtils.class); // NOPMD by marko on 11/24/08 5:02 PM

	/** The Constant CONFIGS_PATH. */

	private static final String CONFIGS_PATH = "/configs";

	/** The schema factory. */

	@Autowired

	private IndexSchemaFactory schemaFactory;

	/** The config factory. */

	private IndexConfigFactory configFactory;

	/** The static configuration classpath. */

	private String staticConfigurationClasspath;

	/**

	 * Upload zookeper config.

	 * 

	 * @param zkClient

	 *            the zk client

	 * @param coreName

	 *            the core name

	 * @param fields

	 *            the fields

	 * @param params

	 *            the params

	 * @param overwrite

	 *            the overwrite

	 */

	public void uploadZookeperConfig(final SolrZkClient zkClient,

			final String coreName,

			final Document fields,

			final Map<String, String> params,

			final boolean overwrite) {

		final String basepath = CONFIGS_PATH + "/" + coreName;

		log.info("uploading solr configuration to ZK for index collection: " + coreName);

		try {

			if (overwrite && zkClient.getSolrZooKeeper().exists(basepath, false) != null) {

				log.info("cleanup ZK configuration: " + coreName);

				for (String child : zkClient.getSolrZooKeeper().getChildren(basepath, false)) {

					final String path = basepath + "/" + child;

					log.debug("cleanup ZK file: " + path);

					zkClient.delete(path, -1, true);

				}

				zkClient.delete(basepath, -1, true);

			}

			if (!zkClient.exists(basepath, true)) {

				log.info("upload ZK configuration: " + coreName);

				zkClient.makePath(basepath, true);

				uploadConfiguration(zkClient, basepath, buildConfiguration(coreName, fields, params));

			}

			log.info("upload ZK configuration complete");

		} catch (Exception e) {

			log.error("unable to upload solr configuration", e);

		}

	}

	/**

	 * Builds the configuration.

	 * 

	 * @param indexName

	 *            the index name

	 * @param fields

	 *            the fields

	 * @param params

	 *            the params

	 * @return the map

	 * @throws IndexServiceException

	 *             the index service exception

	 */

	private Map<String, byte[]> buildConfiguration(final String indexName, final Document fields, final Map<String, String> params)

			throws IndexServiceException {

		Map<String, byte[]> res = Maps.newHashMap();

		try {

			log.debug("adding schema.xml to the resource map");

			res.put("schema.xml", schemaFactory.getSchema(fields).getBytes());

			res.put("solrconfig.xml", configFactory.getConfig(params).getBytes());

			log.debug("adding solrconfig.xml to the resource map");

			PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();

			Resource[] resources = resolver.getResources(getStaticConfigurationClasspath());

			for (Resource r : resources) {

				InputStream is = r.getInputStream();

				if ((r.getFilename() != null) && !r.getFilename().isEmpty()) {

				    res.put(r.getFilename(), IOUtils.toByteArray(is));

				    log.debug("adding " + r.getFilename() +" to the resource map");

				    is.close();

				}

			}

			return res;

		} catch (Throwable e) {

			throw new IndexServiceException("failed to build configuration", e);

		}

	}

	/**

	 * Upload configuration.

	 * 

	 * @param zkClient

	 *            the zk client

	 * @param basePath

	 *            the base path

	 * @param resources

	 *            the resources

	 * @throws KeeperException

	 *             the keeper exception

	 * @throws InterruptedException

	 *             the interrupted exception

	 * @throws IOException

	 *             Signals that an I/O exception has occurred.

	 */

	private void uploadConfiguration(final SolrZkClient zkClient, final String basePath, final Map<String, byte[]> resources) throws KeeperException,

			InterruptedException, IOException {

		if (!zkClient.exists(basePath, true)) {

			zkClient.makePath(basePath, true);

		}

		for (final Entry<String, byte[]> e : resources.entrySet()) {

			String path = basePath + "/" + e.getKey();

			log.debug("upload ZK configuration: " + path);

			zkClient.create(path, e.getValue(), CreateMode.PERSISTENT, true);

		}

	}

	/**

	 * Gets the config factory.

	 * 

	 * @return the config factory

	 */

	public IndexConfigFactory getConfigFactory() {

		return configFactory;

	}

	/**

	 * Sets the config factory.

	 * 

	 * @param configFactory

	 *            the new config factory

	 */

	@Required

	public void setConfigFactory(final IndexConfigFactory configFactory) {

		this.configFactory = configFactory;

	}

	/**

	 * Gets the static configuration classpath.

	 * 

	 * @return the static configuration classpath

	 */

	public String getStaticConfigurationClasspath() {

		return staticConfigurationClasspath;

	}

	/**

	 * Sets the static configuration classpath.

	 * 

	 * @param staticConfigurationClasspath

	 *            the new static configuration classpath

	 */

	@Required

	public void setStaticConfigurationClasspath(final String staticConfigurationClasspath) {

		this.staticConfigurationClasspath = staticConfigurationClasspath;

	}

}

package eu.dnetlib.functionality.index.utils;

import com.google.common.collect.Maps;

import com.google.gson.JsonElement;

import com.google.gson.JsonObject;

import com.google.gson.JsonParser;

import eu.dnetlib.data.provision.index.rmi.IndexServiceException;

import eu.dnetlib.functionality.index.model.Any.ValueType;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.http.HttpResponse;

import org.apache.http.client.HttpClient;

import org.apache.http.client.methods.HttpGet;

import org.apache.solr.client.solrj.SolrClient;

import org.apache.solr.client.solrj.SolrServerException;

import org.apache.solr.client.solrj.impl.CloudSolrClient;

import org.apache.solr.client.solrj.request.LukeRequest;

import org.apache.solr.client.solrj.response.LukeResponse;

import org.apache.solr.client.solrj.response.LukeResponse.FieldInfo;

import org.apache.solr.client.solrj.response.SolrPingResponse;

import org.apache.solr.common.cloud.SolrZkClient;

import org.apache.zookeeper.WatchedEvent;

import org.apache.zookeeper.Watcher;

import org.apache.zookeeper.data.Stat;

import org.springframework.beans.factory.annotation.Required;

import java.io.IOException;

import java.util.Map;

public class RemoteSolrAdministrator {

	/**

	 * The log.

	 */

	private static final Log log = LogFactory.getLog(RemoteSolrAdministrator.class);

	/** The create url request. */

	private static String createURLRequest = "http://%s:%s/solr/admin/collections?action=CREATE&name=%s&numShards=%s&replicationFactor=%s&maxShardsPerNode=%s&collection.configName=%s";

	/** The create url request. */

	private static String reloadURLRequest = "http://%s:%s/solr/admin/collections?action=RELOAD&name=%s";

	/** The http client. */

	private HttpClient httpClient;

	protected Map<String, Map<String, ValueType>> cachedSchema;

	public RemoteSolrAdministrator() {

		this.cachedSchema = Maps.newHashMap();

	}

	public boolean createSolrIndex(final String host,

			final String port,

			final String collectionName,

			final String numShard,

			final String replicationFactor,

			final String maxShardsPerNode,

		final String collectionConfigName) throws IndexServiceException {

		final String uri = generateCreateIndexRequest(host, port, collectionName, numShard, replicationFactor, maxShardsPerNode, collectionConfigName);

		log.info(uri);

		HttpGet request = new HttpGet(uri);

		HttpResponse response;

		try {

			response = getHttpClient().execute(request);

		} catch (Exception e) {

			throw new IndexServiceException("Unable to send request to solr server", e);

		} finally {

			request.releaseConnection();

		}

		if (response.getStatusLine().getStatusCode() != 200)

			throw new IndexServiceException("Error on creating index the error code from solr is" + response.toString());

		return false;

	}

	public boolean indexCollectionExists(final String indexCollectionId, final CloudSolrClient client) {

		Watcher watcher = new Watcher() {

			@Override

			public void process(final WatchedEvent arg0) {}

		};

		try {

			// server.connect();

			SolrZkClient zkClient = client.getZkStateReader().getZkClient();

			if (!zkClient.isConnected()) {

				client.close();

				client.connect();

			}

			byte[] data = zkClient.getData("/clusterstate.json", watcher, new Stat(), true);

			if (data == null) return false;

			String jsonLine = new String(data);

			JsonElement jelement = new JsonParser().parse(jsonLine);

			JsonObject jobject = jelement.getAsJsonObject();

			if (jobject.has(indexCollectionId)) {

				client.setDefaultCollection(indexCollectionId);

				client.connect();

				SolrPingResponse status = client.ping();

				return (Integer) status.getResponseHeader().get("status") == 0;

			} else return false;

		} catch (Exception e) {

			log.error(e);

			return false;

		}

	}

	private ValueType resolveSolrTypeClassName(final String solrTypeName) {

		if (solrTypeName.contains("LongField")) return ValueType.LONG;

		else if (solrTypeName.contains("IntField")) return ValueType.LONG;

		else if (solrTypeName.contains("short")) return ValueType.LONG;

		else if (solrTypeName.contains("float")) return ValueType.DOUBLE;

		else if (solrTypeName.contains("double")) return ValueType.DOUBLE;

		else if (solrTypeName.contains("date")) return ValueType.DATETIME;

		else return ValueType.STRING;

	}

	public Map<String, ValueType> getFieldNamesAndTypes(final String coreName, final SolrClient client) throws IndexServiceException {

		try {

			if (cachedSchema.get(coreName) == null) {

				synchronized (cachedSchema) {

					Map<String, ValueType> schema = readFieldNamesAndTypes(coreName, client);

					log.info("setting cache for schema of collection: " + coreName);

					cachedSchema.put(coreName, schema);

				}

			}

			return cachedSchema.get(coreName);

		} catch (Exception e) {

			throw new IndexServiceException("Unable to get Schema for " + coreName + " exception", e);

		}

	}

	private Map<String, ValueType> readFieldNamesAndTypes(final String coreName, final SolrClient client) throws SolrServerException, IOException {

		final LukeRequest request = new LukeRequest();

		request.setShowSchema(true);

		request.setNumTerms(0);

		final LukeResponse response = request.process(client);

		final Map<String, FieldInfo> fieldInfos = response.getFieldInfo();

		final Map<String, LukeResponse.FieldTypeInfo> fieldTypeInfos = response.getFieldTypeInfo();

		final Map<String, ValueType> result = Maps.newHashMap();

		for (FieldInfo fieldInfo : fieldInfos.values()) {

			LukeResponse.FieldTypeInfo fieldTypeInfo = fieldTypeInfos.get(fieldInfo.getType());

			final String fieldName = fieldTypeInfo.getName().toLowerCase();

			final ValueType fieldType = resolveSolrTypeClassName(fieldName);

			result.put(fieldInfo.getName(), fieldType);

		}

		return result;

	}

	private String generateCreateIndexRequest(final String host,

			final String port,

			final String collectionName,

			final String numShard,

			final String replicationFactor,

			final String collectionConfigName,

			final String maxShardsPerNode) {

		return String.format(createURLRequest, host, port, collectionName, numShard, replicationFactor, maxShardsPerNode, collectionConfigName);

	}

	private String generateUpdateIndexRequest(final String host, final String port, final String collectionName) {

		return String.format(reloadURLRequest, host, port, collectionName);

	}

	/**

	 * @return the httpClient

	 */

	public HttpClient getHttpClient() {

		return httpClient;

	}

	/**

	 * @param httpClient

	 *            the httpClient to set

	 */

	@Required

	public void setHttpClient(final HttpClient httpClient) {

		log.info("setting http client " + httpClient.getClass());

		this.httpClient = httpClient;

	}

	public void reloadCollection(final String host, final String port, final String collectionName) throws IndexServiceException {

		log.info("creating the request of reload index " + generateUpdateIndexRequest(host, port, collectionName));

		HttpGet request = new HttpGet(generateUpdateIndexRequest(host, port, collectionName));

		HttpResponse response;

		try {

			response = httpClient.execute(request);

		} catch (Exception e) {

			throw new IndexServiceException("Unable to send request to solr server", e);

		} finally {

			request.releaseConnection();

		}

		if (response.getStatusLine().getStatusCode() != 200)

			throw new IndexServiceException("Error on reloading index the error code from solr is" + response.toString());

	}

}

package eu.dnetlib.functionality.index;

import com.google.common.collect.Maps;

import eu.dnetlib.data.provision.index.rmi.IndexServiceException;

import eu.dnetlib.functionality.cql.CqlValueTransformerMap;

import eu.dnetlib.functionality.index.feed.DocumentMapperFactory;

import eu.dnetlib.functionality.index.model.Any.ValueType;

import eu.dnetlib.functionality.index.query.IndexQueryFactory;

import eu.dnetlib.functionality.index.query.SolrIndexQueryFactory;

import eu.dnetlib.functionality.index.query.SolrIndexQueryResponseFactory;

import eu.dnetlib.functionality.index.solr.cql.SolrTypeBasedCqlValueTransformerMapFactory;

import eu.dnetlib.functionality.index.solr.feed.SolrDocumentMapperFactory;

import eu.dnetlib.functionality.index.utils.*;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.impl.CloudSolrClient;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

import java.io.IOException;

import java.io.StringReader;

import java.util.Map;

/**

 * The Class SolrIndexServerDAO.

 */

public class SolrIndexServerDAO extends AbstractBackendDescriptor implements IndexServerDAO {

	/**

	 * The log.

	 */

	private static final Log log = LogFactory.getLog(SolrIndexServerDAO.class); // NOPMD by marko on 11/24/08 5:02 PM

	/** The zk utils. */

	@Autowired

	private ZkUtils zkUtils;

	/** The query response factory. */

	@Autowired

	private SolrIndexQueryResponseFactory queryResponseFactory;

	@Autowired

	private SolrIndexQueryFactory solrIndexQueryFactory;

	/** The solr document mapper factory. */

	@Autowired

	private SolrDocumentMapperFactory solrDocumentMapperFactory;

	/** The solr administrator. */

	private RemoteSolrAdministrator solrAdministrator;

	/** The solr type based cql value transformer map. */

	@Autowired

	private SolrTypeBasedCqlValueTransformerMapFactory tMapFactory;

	/**

	 * {@inheritDoc}

	 * 

	 * @see eu.dnetlib.functionality.index.IndexServerDAO#createIndexCollection(eu.dnetlib.functionality.index.utils.MetadataReference,

	 *      java.lang.String)

	 */

	@Override

	public void createIndexCollection(final MetadataReference mdref, final String fields) throws IndexServiceException {

		try(CloudSolrClient client = getClient()) {

			client.connect();

			if (!solrAdministrator.indexCollectionExists(mdref.toString(), client)) {

				Map<String, String> params = Maps.newHashMap();

				final Map<String, String> p = getServiceProperties();

				params.put("numShards", p.get("numShards"));

				params.put("replicationFactor", p.get("replicationFactor"));

				for (IndexConfigFactory.CONFIG_PARAMS param_Name : IndexConfigFactory.CONFIG_PARAMS.values()) {

					params.put(param_Name.toString(), p.get(param_Name.toString()));

				}

				zkUtils.uploadZookeperConfig(client.getZkStateReader().getZkClient(), mdref.toString(), parse(fields), params, true);

				solrAdministrator.createSolrIndex(p.get("host"), p.get("port"), mdref.toString(), p.get("numShards"), p.get("replicationFactor"),

						mdref.toString(), p.get("maxShardsPerNode"));

			}

			client.getZkStateReader().close();

		} catch (Exception e) {

			log.error("Error on creating IndexCollection", e);

			throw new IndexServiceException("Error on creating IndexCollection", e);

		}

	}

	@Override

	public void updateIndexCollection(final MetadataReference mdRef, final Document fields) throws IndexServiceException {

		try(CloudSolrClient client = getClient()) {

			client.connect();

			Map<String, String> params = Maps.newHashMap();

			params.put("numShards", getServiceProperties().get("numShards"));

			params.put("replicationFactor", getServiceProperties().get("replicationFactor"));

			for (IndexConfigFactory.CONFIG_PARAMS param_Name : IndexConfigFactory.CONFIG_PARAMS.values()) {

				params.put(param_Name.toString(), getServiceProperties().get(param_Name.toString()));

			}

			zkUtils.uploadZookeperConfig(client.getZkStateReader().getZkClient(), mdRef.toString(), fields, params, true);

			client.getZkStateReader().close();

			solrAdministrator.reloadCollection(getServiceProperties().get("host"), getServiceProperties().get("port"), mdRef.toString());

		} catch (Exception e) {

			log.error("Error on updating IndexCollection", e);

			throw new IndexServiceException("Error on updating IndexCollection", e);

		}

	}

	/**

	 * Parses the fields parameter.

	 * 

	 * @param fields

	 *            the fields

	 * @return the document

	 * @throws IndexServiceException

	 *             the index service exception

	 */

	private Document parse(final String fields) throws IndexServiceException {

		try {

			return new SAXReader().read(new StringReader(fields));

		} catch (DocumentException e) {

			throw new IndexServiceException("unable to parse fields: " + fields, e);

		}

	}

	/**

	 * {@inheritDoc}

	 * 

	 * @see eu.dnetlib.functionality.index.IndexServerDAO#getIndexCollection(eu.dnetlib.functionality.index.utils.MetadataReference)

	 */

	@Override

	public IndexCollection getIndexCollection(final MetadataReference mdref) throws IndexServiceException {

		CloudSolrClient client = getClient(mdref);

		return new SolrIndexCollection(client);

	}

	/**

	 * {@inheritDoc}

	 * 

	 * @see eu.dnetlib.functionality.index.IndexServerDAO#getSchema(MetadataReference)

	 */

	@Override

	public Map<String, ValueType> getSchema(final MetadataReference mdRef) throws IndexServiceException {

		CloudSolrClient client = getClient(mdRef);

		Map<String, ValueType> fields = solrAdministrator.getFieldNamesAndTypes(mdRef.toString(), client);

		shutdown(mdRef);

		return fields;

	}

	/**

	 * {@inheritDoc}

	 * 

	 * @see eu.dnetlib.functionality.index.IndexServerDAO#getCqlValueTransformerMap(MetadataReference)

	 */

	@Override

	public CqlValueTransformerMap getCqlValueTransformerMap(final MetadataReference mdRef) throws IndexServiceException {

		return tMapFactory.getIt(getSchema(mdRef));

	}

	/**

	 * {@inheritDoc}

	 * 

	 * @see eu.dnetlib.functionality.index.IndexServerDAO#getDocumentMapperFactory()

	 */

	@Override

	public DocumentMapperFactory getDocumentMapperFactory() throws IndexServiceException {

		return solrDocumentMapperFactory;

	}

	/**

	 * {@inheritDoc}

	 * 

	 * @see eu.dnetlib.functionality.index.IndexServerDAO#shutdown(MetadataReference)

	 */

	@Override

	public void shutdown(final MetadataReference mdRef) throws IndexServiceException {

		try {

			getClient(mdRef).close();

		} catch (IOException e) {

			throw new IndexServiceException(e);

		}

	}

	/**

	 * Gets a server with the default collection set according to the given mdRef.

	 * 

	 * @param mdRef

	 *            the md ref

	 * @return a server instance

	 * @throws IndexServiceException

	 *             the index service exception

	 */

	private CloudSolrClient getClient(final MetadataReference mdRef) throws IndexServiceException {

		CloudSolrClient client = getClient();

		client.setDefaultCollection(mdRef.toString());

		return client;

	}

	/**

	 * Gets the server.

	 * 

	 * @return a server instance

	 */

	private CloudSolrClient getClient() {

		String address = getEndpoint().get(ADDRESS);

		log.info("connecting to address: " + address);

		final ZkServers zk = ZkServers.newInstance(address);

		return new CloudSolrClient.Builder(zk.getHosts(), zk.getChroot()).build();

	}

	/**

	 * Gets the solr administrator.

	 * 

	 * @return the solrAdministrator

	 */

	public RemoteSolrAdministrator getSolrAdministrator() {

		return solrAdministrator;

	}

	/**

	 * Sets the solr administrator.

	 * 

	 * @param solrAdministrator

	 *            the solrAdministrator to set

	 */

	@Required

	public void setSolrAdministrator(final RemoteSolrAdministrator solrAdministrator) {

		this.solrAdministrator = solrAdministrator;

	}

	@Override

	public IndexQueryFactory getIndexQueryFactory() {

		return solrIndexQueryFactory;

	}

}

package eu.dnetlib.functionality.index;

import com.google.common.base.Function;

import com.google.common.collect.Iterators;

import com.google.common.collect.Lists;

import eu.dnetlib.data.provision.index.rmi.IndexServiceException;

import eu.dnetlib.functionality.index.model.document.IndexDocument;

import eu.dnetlib.functionality.index.query.SolrIndexDocument;

import eu.dnetlib.functionality.index.utils.IndexFieldUtility;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.impl.CloudSolrClient;

import org.apache.solr.client.solrj.response.UpdateResponse;

import org.apache.solr.common.SolrInputDocument;

import java.io.IOException;

import java.util.Collection;

import java.util.Iterator;

/**

 * The Class SolrIndexCollection.

 */

public class SolrIndexCollection implements IndexCollection {

	/**

	 * The log.

	 */

	private static final Log log = LogFactory.getLog(SolrIndexCollection.class); // NOPMD by marko on 11/24/08 5:02 PM

	/** The Constant STATUS_INDEX_OK. */

	public static final int STATUS_INDEX_OK = 0;

	/** The client. */

	private CloudSolrClient client;

	private boolean shutdown = false;

	/**

	 * The Constructor.

	 *

	 * @param newServer

	 *            the client

	 */

	public SolrIndexCollection(final CloudSolrClient newServer) {

		this.client = newServer;

		client.connect();

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.IndexCollection#add(eu.dnetlib.functionality.index.model.document.IndexDocument)

	 */

	@Override

	public boolean add(final IndexDocument doc) throws IndexServiceException {

		if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");

		final SolrIndexDocument solrDocument = (SolrIndexDocument) doc;

		try {

			final UpdateResponse response = client.add(solrDocument.getSolrDocument());

			return response.getStatus() == 0;

		} catch (final Exception e) {

			throw new IndexServiceException("Unable to add document", e);

		}

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.IndexCollection#addAll(java.util.Iterator)

	 */

	@Override

	public boolean addAll(final Iterator<IndexDocument> docs) throws IndexServiceException {

		if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");

		final Iterator<SolrInputDocument> solrDocs = Iterators.transform(docs, new Function<IndexDocument, SolrInputDocument>() {

			@Override

			public SolrInputDocument apply(final IndexDocument doc) {

				final SolrIndexDocument solrDocument = (SolrIndexDocument) doc;

				return solrDocument.getSolrDocument();

			}

		});

		try {

			final UpdateResponse response = client.add(Lists.newArrayList(solrDocs));

			return response.getStatus() == 0;

		} catch (final Exception e) {

			throw new IndexServiceException("Unable to add document", e);

		}

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.IndexCollection#addAll(java.util.Collection)

	 */

	@Override

	public boolean addAll(final Collection<IndexDocument> docs) throws IndexServiceException {

		if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");

		return addAll(docs.iterator());

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.IndexCollection#deleteIndex(java.lang.String)

	 */

	@Override

	public boolean deleteIndex(final String dsId) throws IndexServiceException {

		if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");

		return doDelete(IndexFieldUtility.DS_ID + " : \"" + dsId + "\"");

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.IndexCollection#deleteByQuery(java.lang.String, java.lang.String)

	 */

	@Override

	public boolean deleteByQuery(final String query, final String dsId) throws IndexServiceException {

		if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");

		if (StringUtils.isBlank(dsId)) return doDelete(query);

		return doDelete(query + " AND " + IndexFieldUtility.DS_ID + " : \"" + dsId + "\"");

	}

	/**

	 * Do delete.

	 *

	 * @param query

	 *            the query

	 * @return true, if do delete

	 * @throws IndexServiceException

	 *             the index service exception

	 */

	protected boolean doDelete(final String query) throws IndexServiceException {

		if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");

		try {

			log.debug("delete by query: " + query);

			return client.deleteByQuery(query).getStatus() == STATUS_INDEX_OK;

		} catch (final Exception e) {

			throw new IndexServiceException("unable to run delete by query: " + query, e);

		}

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.IndexCollection#commit()

	 */

	@Override

	public boolean commit() throws IndexServiceException {

		if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");

		try {

			log.info("performing commit");

			final UpdateResponse rsp = client.commit();

			log.info(String.format("commit completed in %s, status %s", rsp.getElapsedTime(), rsp.getStatus()));

			return rsp.getStatus() == STATUS_INDEX_OK;

		} catch (final Throwable e) {

			throw new IndexServiceException("unable to perform index commit", e);

		}

	}

	@Override

	public void shutdown() {

		try {

			client.close();

		} catch (IOException e) {

			throw new RuntimeException(e);

		}

		shutdown = true;

	}

	public boolean isShutdown() {

		return shutdown;

	}

	public void setShutdown(final boolean shutdown) {

		this.shutdown = shutdown;

	}

}

service.solr.index.staticConfigurationClasspath= eu/dnetlib/functionality/index/conf/files/**

service.solr.index.jsonConfiguration= \

	{"id":"solr",\

	"address":"localhost:9983",\

	"port":"8983",\

	"webContext":"solr",\

	"numShards":"1",\

	"replicationFactor":"1",\

    "maxShardsPerNode":"2",\

	"host":"localhost",\

	"feedingShutdownTolerance":"30000",\

	"feedingBufferFlushThreshold":"1000",\

	"feedingSimulationMode":"false",\

	"luceneMatchVersion":"4.9",\

	"serverLibPath":"../../../../contrib/extraction/lib",\

	"filterCacheSize":"512","filterCacheInitialSize":"512",\

	"queryCacheSize":"512","queryCacheInitialSize":"512",\

	"documentCacheSize":"512","documentCacheInitialSize":"512",\

	"ramBufferSizeMB":"960","mergeFactor":"40",\

	"autosoftcommit":"-1","autocommit":"15000",\

	"termIndexInterval":"1024","maxIndexingThreads":"8",\

	"queryResultWindowSize":"20","queryResultMaxDocCached":"200"} 

service.index.solr.returnEmptyFields		= true

service.index.solr.schema.textfieldtype		= text_common

service.index.solr.nh.mdformat.enable		= true

service.index.solr.highlight.enable			= true

service.index.solr.rank.enable				= true

service.index.feed.repeatDelay				= 20000

services.mapreduce.index.solr.feed.sim.mode	= false

service.index.solr.document.factory			= eu.dnetlib.functionality.index.solr.feed.StreamingInputDocumentFactory

service.index.solr.schema.template			= classpath:/eu/dnetlib/functionality/index/conf/schemaTemplate.xslt

<!-- admin-extra.menu-top.html -->

/*

  This is a basic skeleton JavaScript update processor.

  In order for this to be executed, it must be properly wired into solrconfig.xml; by default it is commented out in

  the example solrconfig.xml and must be uncommented to be enabled.

  See http://wiki.apache.org/solr/ScriptUpdateProcessor for more details.

*/

function processAdd(cmd) {

  doc = cmd.solrDoc;  // org.apache.solr.common.SolrInputDocument

  id = doc.getFieldValue("id");

  logger.info("update-script#processAdd: id=" + id);

// Set a field value:

//  doc.setField("foo_s", "whatever");

// Get a configuration parameter:

//  config_param = params.get('config_param');  // "params" only exists if processor configured with <lst name="params">

// Get a request parameter:

// some_param = req.getParams().get("some_param")

// Add a field of field names that match a pattern:

//   - Potentially useful to determine the fields/attributes represented in a result set, via faceting on field_name_ss

//  field_names = doc.getFieldNames().toArray();

//  for(i=0; i < field_names.length; i++) {

//    field_name = field_names[i];

//    if (/attr_.*/.test(field_name)) { doc.addField("attribute_ss", field_names[i]); }

//  }

}

function processDelete(cmd) {

  // no-op

}

function processMergeIndexes(cmd) {

  // no-op

}

function processCommit(cmd) {

  // no-op

}

function processRollback(cmd) {

  // no-op

}

function finish() {

  // no-op

}

<!-- admin-extra.menu-bottom.html -->

# The ASF licenses this file to You under the Apache License, Version 2.0

# (the "License"); you may not use this file except in compliance with

# the License.  You may obtain a copy of the License at

#

#     http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

#-----------------------------------------------------------------------

# Use a protected word file to protect against the stemmer reducing two

# unrelated words to the same base word.

# Some non-words that normally won't be encountered,

# just to test that they won't be stemmed.

dontstems

zwhacky

# Licensed to the Apache Software Foundation (ASF) under one or more

# contributor license agreements.  See the NOTICE file distributed with

# this work for additional information regarding copyright ownership.

# The ASF licenses this file to You under the Apache License, Version 2.0

# (the "License"); you may not use this file except in compliance with

# the License.  You may obtain a copy of the License at

#

#     http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

<?xml version="1.0" encoding="UTF-8" ?>

<!--

 Licensed to the Apache Software Foundation (ASF) under one or more

 contributor license agreements.  See the NOTICE file distributed with

 this work for additional information regarding copyright ownership.

 The ASF licenses this file to You under the Apache License, Version 2.0

 (the "License"); you may not use this file except in compliance with

 the License.  You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software

 distributed under the License is distributed on an "AS IS" BASIS,

 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 See the License for the specific language governing permissions and

 limitations under the License.

-->

<!-- If this file is found in the config directory, it will only be

     loaded once at startup.  If it is found in Solr's data

     directory, it will be re-loaded every commit.

   See http://wiki.apache.org/solr/QueryElevationComponent for more info

-->

<elevate>

 <query text="foo bar">

  <doc id="1" />

  <doc id="2" />

  <doc id="3" />

 </query>

 <query text="ipod">

   <doc id="MA147LL/A" />  <!-- put the actual ipod at the top -->

   <doc id="IW-02" exclude="true" /> <!-- exclude this cable -->

 </query>

</elevate>

# Licensed to the Apache Software Foundation (ASF) under one or more

# contributor license agreements.  See the NOTICE file distributed with

# this work for additional information regarding copyright ownership.

# The ASF licenses this file to You under the Apache License, Version 2.0