D-Net

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-index-client</artifactId>

	<packaging>jar</packaging>

	<version>1.1.0</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-index-client/tags/dnet-index-client-1.1.0</developerConnection>

	</scm>

	<dependencies>

	<dependency>

			<groupId>org.springframework</groupId>

			<artifactId>spring-beans</artifactId>

			<version>${spring.version}</version>

		</dependency>		

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.mockito</groupId>

			<artifactId>mockito-core</artifactId>

			<version>1.6</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>com.google.code.gson</groupId>

			<artifactId>gson</artifactId>

			<version>${google.gson.version}</version>		

		</dependency>		

		<dependency>

			<groupId>com.mycila</groupId>

			<artifactId>xmltool</artifactId>

			<version>3.3</version>

		</dependency>	

		<dependency>

		<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-service-common</artifactId>

			<version>[1.0.0,2.0.0)</version>	

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-cql-utils</artifactId>

			<version>[1.0.0,2.0.0)</version>		

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-data-provision-rmi</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

	</dependencies>

</project>

package eu.dnetlib.functionality.index.client;

import java.util.List;

import eu.dnetlib.functionality.index.client.response.BrowseEntry;

import eu.dnetlib.functionality.index.client.response.LookupResponse;

public interface IndexClient {

	LookupResponse lookup(String query, List<String> filterQuery, int from, int to) throws IndexClientException;

	List<BrowseEntry> browse(String query, List<String> browseFields, int max) throws IndexClientException;

	List<BrowseEntry> browse(String query, List<String> browseFields, int max, List<String> filterQuery) throws IndexClientException;

	long delete(String query) throws IndexClientException;

	void stop() throws IndexClientException;

}

package eu.dnetlib.functionality.index.client;

import java.util.Map;

import eu.dnetlib.functionality.index.cql.CqlValueTransformerMap;

import eu.dnetlib.functionality.index.query.IndexQueryFactory;

import eu.dnetlib.functionality.index.utils.MetadataReference;

public abstract class AbstractIndexClient implements IndexClient {

	protected static String INDEX_HOST = "host";

	protected static String INDEX_PORT = "port";

	protected static String INDEX_BACKEND_ID = "id";

	protected static String ZK_ADDRESS = "address";

	/** The format. */

	private String format;

	/** The layout. */

	private String layout;

	/** The interpretation. */

	private String interpretation;

	protected Map<String, String> serviceProperties;

	public AbstractIndexClient(final String format, final String layout, final String interpretation, final Map<String, String> serviceProperties) {

		this.format = format;

		this.layout = layout;

		this.interpretation = interpretation;

		this.serviceProperties = serviceProperties;

	}

	public abstract CqlValueTransformerMap getCqlValueTransformerMap(MetadataReference mdRef) throws IndexClientException;

	public abstract IndexQueryFactory getIndexQueryFactory();

	/**

	 * Gets the format.

	 *

	 * @return the format

	 */

	public String getFormat() {

		return format;

	}

	/**

	 * Sets the format.

	 *

	 * @param format

	 *            the format

	 */

	public void setFormat(final String format) {

		this.format = format;

	}

	/**

	 * Gets the layout.

	 *

	 * @return the layout

	 */

	public String getLayout() {

		return layout;

	}

	/**

	 * Sets the layout.

	 *

	 * @param layout

	 *            the layout

	 */

	public void setLayout(final String layout) {

		this.layout = layout;

	}

	/**

	 * Gets the interpretation.

	 *

	 * @return the interpretation

	 */

	public String getInterpretation() {

		return interpretation;

	}

	/**

	 * Sets the interpretation.

	 *

	 * @param interpretation

	 *            the interpretation

	 */

	public void setInterpretation(final String interpretation) {

		this.interpretation = interpretation;

	}

	public Map<String, String> getServiceProperties() {

		return serviceProperties;

	}

	public void setServiceProperties(final Map<String, String> serviceProperties) {

		this.serviceProperties = serviceProperties;

	}

}

package eu.dnetlib.functionality.index.client;

import org.springframework.beans.factory.annotation.Autowired;

import eu.dnetlib.functionality.index.utils.ServiceTools;

/**

 * The Class AbstractIndexClientFactory.

 */

public abstract class AbstractIndexClientFactory implements IndexClientFactory {

	/** The backend id. */

	private String backendId;

	@Autowired

	protected ServiceTools isQueryTools;

	/**

	 * Inits the class.

	 */

	public abstract void init() throws IndexClientException;

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.functionality.index.client.IndexClientFactory#getBackendId()

	 */

	@Override

	public String getBackendId() {

		return backendId;

	}

	/**

	 * Sets the backend id.

	 *

	 * @param backendId

	 *            the backend id

	 */

	public void setBackendId(final String backendId) {

		this.backendId = backendId;

	}

}

package eu.dnetlib.functionality.index.client;

import eu.dnetlib.data.provision.index.rmi.IndexServiceException;

public class IndexClientException extends IndexServiceException {

	/**

	 *

	 */

	private static final long serialVersionUID = 1851955470714206331L;

	public IndexClientException() {

		super();

	}

	public IndexClientException(final String message) {

		super(message);

	}

	public IndexClientException(final Throwable cause) {

		super(cause);

	}

	public IndexClientException(final String message, final Throwable cause) {

		super(message, cause);

	}

}

package eu.dnetlib.functionality.index.query;

import java.util.HashMap;

import java.util.Map;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import eu.dnetlib.functionality.index.client.IndexClientException;

import eu.dnetlib.functionality.index.utils.MDFormatReader;

import eu.dnetlib.functionality.index.utils.MetadataReference;

import eu.dnetlib.functionality.index.utils.ServiceTools;

public class Weights extends HashMap<MetadataReference, Map<String, String>> {

	private static final long serialVersionUID = -3517914310574484765L;

	private static final Log log = LogFactory.getLog(Weights.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Autowired

	private ServiceTools serviceTools;

	@Autowired

	private MDFormatReader mdFormatReader;

	public Weights() {

		super();

	}

	public void initialize() throws IndexClientException {

		log.info("initializing weights");

		for (MetadataReference mdRef : serviceTools.listMDRefs()) {

			put(mdRef, mdFormatReader.getAttributeMap(mdRef, "weight"));

		}

		log.info("weights initialization completed");

	}

	@Override

	public Map<String, String> put(final MetadataReference mdRef, final Map<String, String> w) {

		log.info("[" + mdRef + "]" + " adding weights: " + w);

		return super.put(mdRef, w);

	}

	@Override

	public Map<String, String> get(final Object mdRef) {

		Map<String, String> map = super.get(mdRef);

		return map != null ? map : initAndGet(mdRef);

	}

	private Map<String, String> initAndGet(final Object mdRef) {

		try {

			initialize();

		} catch (IndexClientException e) {

			throw new RuntimeException(e);

		}

		return super.get(mdRef);

	}

}

package eu.dnetlib.functionality.index.client;

import java.util.List;

import javax.annotation.Resource;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.tools.ServiceLocator;

/**

 * The Class GlobalIndexClientFactory.

 */

/**

 * The Class ResolvingIndexClientFactory.

 */

public class ResolvingIndexClientFactory {

	/** The index client factories. */

	@Resource

	private List<IndexClientFactory> indexClientFactories;

	/** The lookup locator. */

	@Resource(name = "lookupLocator")

	private ServiceLocator<ISLookUpService> lookupLocator;

	/**

	 * Gets the client.

	 *

	 * @param format

	 *            the format

	 * @param layout

	 *            the layout

	 * @param interpretation

	 *            the interpretation

	 * @param backendId

	 *            the backend id

	 * @return the client

	 * @throws IndexClientException

	 *             the index client exception

	 */

	public IndexClient getClient(final String format, final String layout, final String interpretation, final String backendId) throws IndexClientException {

		for (IndexClientFactory factory : indexClientFactories) {

			if (factory.getBackendId().equalsIgnoreCase(backendId)) return factory.getClient(format, layout, interpretation);

		}

		throw new IndexClientException("No client for bakendId: " + backendId);

	}

}

package eu.dnetlib.functionality.index.client;

public interface IndexClientFactory {

	String getBackendId();

	IndexClient getClient(String format, String layout, String interpretation) throws IndexClientException;

}

package eu.dnetlib.functionality.index.client.response;

import java.util.List;

import javax.xml.bind.annotation.XmlRootElement;

import eu.dnetlib.functionality.index.query.QueryResponseParser;

@XmlRootElement

public class LookupResponse {

	private long total;

	private long from;

	private long to;

	private List<String> records;

	public LookupResponse() {}

	public LookupResponse(final QueryResponseParser parser) {

		this.total = parser.getNumFound();

		this.from = parser.getStart();

		this.to = parser.getStart() + parser.getResults().size();

		this.records = parser.getResults();

	}

	public LookupResponse(final long total, final long from, final long to, final List<String> records) {

		this.total = total;

		this.from = from;

		this.to = to;

		this.records = records;

	}

	public long getTotal() {

		return total;

	}

	public void setTotal(final long total) {

		this.total = total;

	}

	public long getFrom() {

		return from;

	}

	public void setFrom(final long from) {

		this.from = from;

	}

	public long getTo() {

		return to;

	}

	public void setTo(final long to) {

		this.to = to;

	}

	public List<String> getRecords() {

		return records;

	}

	public void setRecords(final List<String> records) {

		this.records = records;

	}

}

package eu.dnetlib.functionality.index.client.response;

import javax.xml.bind.annotation.XmlRootElement;

@XmlRootElement

public class BrowseValueEntry implements Comparable<BrowseValueEntry> {

	private String value;

	private int size;

	public BrowseValueEntry() {}

	public BrowseValueEntry(final String value, final int size) {

		this.value = value;

		this.size = size;

	}

	public String getValue() {

		return value;

	}

	public void setValue(final String value) {

		this.value = value;

	}

	public int getSize() {

		return size;

	}

	public void setSize(final int size) {

		this.size = size;

	}

	@Override

	public int compareTo(final BrowseValueEntry e) {

		return Integer.compare(getSize(), e.getSize());

	}

}

package eu.dnetlib.functionality.index.client.response;

import java.util.List;

public class BrowseEntry {

	private String field;

	private String label;

	private List<BrowseValueEntry> values;

	public BrowseEntry() {}

	public BrowseEntry(final String field, final String label,final  List<BrowseValueEntry> values) {

		this.field = field;

		this.label = label;

		this.values = values;

	}

	public String getField() {

		return field;

	}

	public void setField(final String field) {

		this.field = field;

	}

	public String getLabel() {

		return label;

	}

	public void setLabel(final String label) {

		this.label = label;

	}

	public List<BrowseValueEntry> getValues() {

		return values;

	}

	public void setValues(final List<BrowseValueEntry> values) {

		this.values = values;

	}

}

package eu.dnetlib.functionality.index.feed;

import java.util.concurrent.Callable;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import eu.dnetlib.functionality.index.model.document.IndexDocument;

import eu.dnetlib.functionality.index.utils.IndexFieldUtility;

/**

 * The Class DocumentFeeder.

 */

public class DocumentFeeder implements Callable<FeedResult> {

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(DocumentFeeder.class);

	/** The index collection. */

	private IndexFeederCollection indexCollection;

	/** The records. */

	private Iterable<IndexDocument> records;

	/**

	 * Instantiates a new document feeder.

	 *

	 * @param indexCollection

	 *            the index collection

	 * @param records

	 *            the records

	 */

	public DocumentFeeder(final IndexFeederCollection indexCollection, final Iterable<IndexDocument> records) {

		this.indexCollection = indexCollection;

		this.records = records;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see java.util.concurrent.Callable#call()

	 */

	@Override

	public FeedResult call() throws Exception {

		final FeedResult res = new FeedResult(System.currentTimeMillis());

		for (IndexDocument doc : records) {

			boolean rsp;

			switch (doc.getStatus()) {

			case OK:

				rsp = indexCollection.add(doc);

				if (rsp) {

					res.add();

				} else {

					res.mark();

				}

				break;

			case MARKED:

				res.mark();

				log.debug("skipping record: " + doc.getFieldValue(IndexFieldUtility.INDEX_RECORD_ID));

				break;

			case ERROR:

				res.skip();

				log.info("Error on record: " + doc.getFieldValue(IndexFieldUtility.INDEX_RECORD_ID));

				break;

			default:

				throw new IllegalStateException("unknow document status");

			}

		}

		return res;

	}

}

package eu.dnetlib.functionality.index.feed;

/**

 * FeedResult helps verifying feeding operations

 * 

 * @author claudio

 * 

 */

public class FeedResult {

	/**

	 * counter of added documents.

	 */

	private Integer added = null;

	/**

	 * counter of skipped documents

	 */

	private Integer skipped = null;

	/**

	 * counter of marked-for-deletion documents.

	 */

	private Integer marked = null;

	/**

	 * marks the start of feed process.

	 */

	private long timeStart;

	/**

	 * time elapsed to complete the feeding.

	 */

	private long timeElapsed;

	/**

	 * builds a new FeedResult

	 * 

	 * @param timeStart

	 *            the start time of feed process.

	 */

	public FeedResult(final long timeStart) {

		this.added = 0;

		this.skipped = 0;

		this.marked = 0;

		this.timeStart = timeElapsed = timeStart;

	}

	/**

	 * increments added counter.

	 */

	public void add() {

		added++;

	}

	/**

	 * increments marked counter.

	 */

	public void mark() {

		marked++;

	}

	/**

	 * increments skipped counter.

	 */

	public void skip() {

		skipped++;

	}

	public int getAdded() {

		return added;

	}

	public int getSkipped() {

		return skipped;

	}

	public int getMarked() {

		return marked;

	}

	/**

	 * method calculates the time elapsed since the feed process has started.

	 * 

	 * @return the time elapsed since the feed process has started.

	 */

	public long getTime() {

		if (timeElapsed > 0) return timeElapsed - timeStart;

		return System.currentTimeMillis() - timeStart;

	}

	public FeedResult setTimeElapsed(final long timeElapsed) {

		this.timeElapsed = timeElapsed;

		return this;

	}

	@Override

	public String toString() {

		return "[added: " + getAdded() + " skipped: " + getSkipped() + " marked: " + getMarked() + " time: " + (getTime() / 1000) + " sec]";

	}

}

package eu.dnetlib.functionality.index.feed;

/**

 * Allowed feed mode.

 * 

 * @author claudio

 * 

 */

public enum FeedMode {

	REFRESH, INCREMENTAL;

};

package eu.dnetlib.functionality.index.feed;

import java.util.Map;

import com.google.common.base.Function;

import eu.dnetlib.functionality.index.model.Any.ValueType;

import eu.dnetlib.functionality.index.model.document.IndexDocument;

import eu.dnetlib.functionality.index.utils.MetadataReference;

/**

 * A factory for creating DocumentMapper objects.

 */

public interface DocumentMapperFactory {

	/**

	 * Gets the record mapper.

	 * 

	 * @param schema

	 *            the schema

	 * @param mdRef

	 *            the md ref

	 * @param dsId

	 *            the ds id

	 * @param version

	 *            the version

	 * @return the record mapper

	 */

	public Function<String, IndexDocument> getRecordMapper(final Map<String, ValueType> schema,

			final MetadataReference mdRef,

			final String dsId,

			final String version);

}

package eu.dnetlib.functionality.index.feed;

import java.util.Collection;

import java.util.Iterator;

import eu.dnetlib.data.provision.index.rmi.IndexServiceException;

import eu.dnetlib.functionality.index.model.document.IndexDocument;

public interface IndexFeederCollection {

	/**

	 * Adds an input Document to the index.

	 *

	 * @param doc

	 *            the doc

	 * @return the solr update response

	 * @throws IndexServiceException

	 *             the index service exception

	 */

	public boolean add(final IndexDocument doc) throws IndexServiceException;

	/**

	 * Adds all the documents to the index giving an iterator of input documents.

	 *

	 * @param docs

	 *            the iterator of input documents

	 * @return the solr update response

	 * @throws IndexServiceException

	 *             the index service exception

	 */

	public boolean addAll(final Iterator<IndexDocument> docs) throws IndexServiceException;

	/**

	 * Adds the all.

	 *

	 * @param docs

	 *            the docs

	 * @return the update response

	 * @throws IndexServiceException

	 *             the index service exception

	 */

	public boolean addAll(final Collection<IndexDocument> docs) throws IndexServiceException;

	/**

	 * Delete index.

	 *

	 * @param dsId

	 *            the ds id

	 * @return true, if successful

	 * @throws IndexServiceException

	 *             the index service exception

	 */

	public boolean deleteIndex(final String dsId) throws IndexServiceException;

	/**

	 * Delete data from the index by query.

	 *

	 * @param query

	 *            the query

	 * @param dsId

	 *            the ds id

	 * @return true, if successful

	 * @throws IndexServiceException

	 *             the index service exception

	 */

	public boolean deleteByQuery(final String query, final String dsId) throws IndexServiceException;

	/**

	 * Commit the data.

	 *

	 * @return true, if successful

	 * @throws IndexServiceException

	 *             the index service exception

	 */

	public boolean commit() throws IndexServiceException;

}

package eu.dnetlib.functionality.index.utils;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import javax.annotation.Resource;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import eu.dnetlib.data.provision.index.rmi.IndexServiceException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpDocumentNotFoundException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.is.registry.ISRegistryDocumentNotFoundException;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;

import eu.dnetlib.enabling.tools.ServiceLocator;

import eu.dnetlib.functionality.index.client.IndexClientException;

public class ServiceTools {

	private static final Log log = LogFactory.getLog(ServiceTools.class);

	/** The lookup locator. */

	@Resource(name = "lookupLocator")

	private ServiceLocator<ISLookUpService> lookupLocator;

	/**

	 * registry locator.

	 */

	private ServiceLocator<ISRegistryService> registryLocator;

	@Resource

	private MetadataReferenceFactory mdFactory;

	public List<MetadataReference> listMDRefs() throws IndexClientException {

		return Lists.newArrayList(Iterables.transform(listMDRefsAsString(), new Function<String, MetadataReference>() {

			@Override

			public MetadataReference apply(final String s) {

				return mdFactory.decodeMetadata(s);

			}

		}));

	}

	private List<String> quickSearchProfile(final String xquery) throws IndexClientException {

		try {

			return lookupLocator.getService().quickSearchProfile(xquery);

		} catch (ISLookUpException e) {

			throw new IndexClientException(e);

		}

	}

	public MetadataReference getMetadataRef(final String dsId) throws IndexServiceException {

		final String xquery = "for $x in //RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value='" + dsId + "']//CONFIGURATION " + "return concat("

				+ "$x/METADATA_FORMAT/text(),'-'," + "$x/METADATA_FORMAT_LAYOUT/text(),'-'," + "$x/METADATA_FORMAT_INTERPRETATION/text())";

		return mdFactory.decodeMetadata(getResourceProfileByQuery(xquery));

	}

	private String getResourceProfileByQuery(final String xquery) throws IndexServiceException {

		try {

			return lookupLocator.getService().getResourceProfileByQuery(xquery);

		} catch (ISLookUpDocumentNotFoundException e) {

			throw new IndexServiceException(e);

		} catch (ISLookUpException e) {

			throw new IndexServiceException(e);

		}

	}

	public String getIndexFields(final String dsId) throws IndexServiceException {

		return getIndexFields(getMetadataRef(dsId));

	}

	public String getIndexFields(final MetadataReference mdRef) {

		final String xquery = "for $x in collection('')/RESOURCE_PROFILE/BODY[CONFIGURATION/NAME='" + mdRef.getFormat()

				+ "'] return $x/STATUS/LAYOUTS/LAYOUT[@name='" + mdRef.getLayout() + "']/FIELDS";

		try {

			return getResourceProfileByQuery(xquery);

		} catch (IndexServiceException e) {

			log.warn("couldn't find Metadata format profile matching specs: " + mdRef.toString());

			return "";

		}

	}

	public List<String> listDsIds() throws IndexClientException {

		final String xquery = "//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='IndexDSResourceType']//RESOURCE_IDENTIFIER/@value/string()";

		return quickSearchProfile(xquery);

	}

	private List<String> listMDRefsAsString() throws IndexClientException {

		final String xquery = "for $x in //RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='MDFormatDSResourceType'] "

				+ "let $format:= $x//CONFIGURATION/NAME/string() " + "for $y in $x//LAYOUTS/LAYOUT " + "let $layout:= $y//LAYOUT/@name/string() "

				+ "let $interpretation:= $x//CONFIGURATION/INTERPRETATION/text() " + "return concat($format,'-',$layout,'-',$interpretation) ";

		return quickSearchProfile(xquery);

	}

	public Map<String, String> getIndexProperties(final String backendId) throws IndexClientException {

		String query = "for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value=\"IndexServiceResourceType\"]//SERVICE_PROPERTIES/PROPERTY"

				+ " return concat($x/@key/string(),\":::\", $x/@value/string())";

		Map<String, String> indexProperties = new HashMap<String, String>();

		try {

			List<String> results = lookupLocator.getService().quickSearchProfile(query);

			if (results != null) {

				for (String s : results) {

					String[] values = s.split(":::");

					if ((values != null) && (values.length == 2)) {

						String key = values[0];

						String value = values[1];

						if (StringUtils.startsWith(key, backendId)) {

							indexProperties.put(StringUtils.substringAfter(key, backendId + ":"), value);

						}

					}

				}

			}

			return indexProperties;

		} catch (ISLookUpException e) {

			throw new IndexClientException();

		}

	}

	public String registerProfile(final String resourceProfile) throws IndexServiceException {

		try {

			return getRegistryLocator().getService().registerProfile(resourceProfile);

		} catch (ISRegistryException e) {

			throw new IndexServiceException(e);

		}

	}

	@Required

	public void setRegistryLocator(final ServiceLocator<ISRegistryService> registryLocator) {

		this.registryLocator = registryLocator;

	}

	public ServiceLocator<ISRegistryService> getRegistryLocator() {

		return registryLocator;

	}

	public boolean incrementHandledDataStructures(final String backendId) throws IndexServiceException {

		final String xquery = "let $x := //RESOURCE_PROFILE[HEADER/PROTOCOLS/PROTOCOL/@name='" + backendId + "'],"

				+ "$tot := $x//STATUS/HANDLED_DATASTRUCTURE/number() + 1 " + "return update replace $x//STATUS/HANDLED_DATASTRUCTURE with "

				+ "<HANDLED_DATASTRUCTURE>{$tot}</HANDLED_DATASTRUCTURE>";

		log.info("performing increment of HANDLED_DATASTRUCTURE");

		return executeXUpdate(xquery);

	}

	private boolean executeXUpdate(final String xquery) throws IndexServiceException {

		try {

			return getRegistryLocator().getService().executeXUpdate(xquery);

		} catch (ISRegistryException e) {

			throw new IndexServiceException(e);

		}

	}

	public String getServiceAddress(final String backendId) {

		final String xquery = "let $x := //RESOURCE_PROFILE[HEADER/PROTOCOLS/PROTOCOL/@name='" + backendId + "']"

				+ "return $x//PROTOCOL[./@name='SOAP']/@address/string()";

		try {

			return getResourceProfileByQuery(xquery);

		} catch (IndexServiceException e) {

			log.warn("couldn't find service Address for index Service with protocol: " + backendId);

			return "";

		}

	}

	public boolean deleteIndexDS(final String dsId) throws IndexServiceException {

		try {

			return getRegistryLocator().getService().deleteProfile(dsId);

		} catch (ISRegistryDocumentNotFoundException e) {

			throw new IndexServiceException(e);

		} catch (ISRegistryException e) {

			throw new IndexServiceException(e);

		}

	}

	public List<String> getBackendIds(final MetadataReference mdRef) throws IndexServiceException {

		String query = "distinct-values(//RESOURCE_PROFILE[.//METADATA_FORMAT='%s' and .//METADATA_FORMAT_LAYOUT='%s' and .//METADATA_FORMAT_INTERPRETATION='%s']//BACKEND/@ID/string())";

		try {

			String instanceQuery = String.format(query, mdRef.getFormat(), mdRef.getLayout(), mdRef.getInterpretation());

			log.debug("Executing query to IS: " + instanceQuery);

			return lookupLocator.getService().quickSearchProfile(instanceQuery);

		} catch (ISLookUpException e) {

			throw new IndexServiceException(e);

		}

	}

}

package eu.dnetlib.functionality.index.utils;

/**

 * IndexMap keeps track of IndexDataStructure-to-physical index mappings and vice-versa.

 * 

 * @author claudio

 * 

 */

public class IndexFieldUtility {

	public static final String INDEX_FIELD_PREFIX = "__";

	public static final String INDEX_DSID_ALL = "ALL";

	public static final String SCORE_FIELD = "score";

	public static final String DS_ID = INDEX_FIELD_PREFIX + "dsid";

	public static final String DS_VERSION = INDEX_FIELD_PREFIX + "dsversion";

	public static final String RESULT = INDEX_FIELD_PREFIX + "result";

	public static final String ALL_FIELDS = INDEX_FIELD_PREFIX + "all";

	public static final String DELETE_DOCUMENT = INDEX_FIELD_PREFIX + "deleted";

	public static final String INDEX_RECORD_ID = INDEX_FIELD_PREFIX + "indexrecordidentifier";

	public static final String FULLTEXT_ID = INDEX_FIELD_PREFIX + "fulltext";

	public static final String INDEXNAME_PARAM = INDEX_FIELD_PREFIX + "indexName";

	public static final String queryAll = "*:*";

	public static final String FIELD_NAME = "name";

	public static final String FIELD_BROWSING_ALIAS_FOR = "browsingAliasFor";

	public static final String XPATH_BROWSING_ALIAS_FOR = "//FIELD[@" + FIELD_BROWSING_ALIAS_FOR + "]";

}

package eu.dnetlib.functionality.index.utils;

import com.google.common.collect.ComparisonChain;

/**

 * Container class for metadata format, layout and interpretation.

 * 

 * @author claudio

 *

 */

public class MetadataReference {

	/**

	 * Metadata format.

	 */

	private String format;

	/**

	 * Metadata layout.

	 */

	private String layout;

	/**

	 * Metadata interpretation.

	 */

	private String interpretation;

	/**

	 * Constructor for MetadataReference.

	 * 

	 * @param format

	 * 			Metadata format

	 * @param layout

	 * 			Metadata layout

	 * @param interpretation

	 * 			Metadata interpretation

	 */

	public MetadataReference(final String format, final String layout, final String interpretation) {

		this.format = format;

		this.layout = layout;

		this.interpretation = interpretation;

	}

	public String getFormat() {

		return format;

	}

	public String getLayout() {

		return layout;

	}

	public String getInterpretation() {

		return interpretation;

	}

	@Override

	public boolean equals(final Object that) {

		if (!(that instanceof MetadataReference))

			return false;

		final MetadataReference mdRef = (MetadataReference) that;

		return ComparisonChain.start()

	        .compare(this.format, mdRef.getFormat())

	        .compare(this.layout, mdRef.getLayout())

	        .compare(this.interpretation, mdRef.getInterpretation())

	        .result() == 0;		

	}

	@Override

	public int hashCode() {

		return getFormat().hashCode() + getLayout().hashCode() + getInterpretation().hashCode();

	}	

	@Override

	public String toString() {

		return getFormat() + "-" + getLayout() + "-" + getInterpretation();

	}

}

package eu.dnetlib.functionality.index.utils;

public class MetadataReferenceFactory {

	public MetadataReference getMetadata(final String format, final String layout, final String interpretation) {

		return new MetadataReference(format, layout, interpretation);

	}

	public MetadataReference decodeMetadata(final String encoded) {

		String[] split = encoded.split("-");

		if (split.length == 3) return getMetadata(split[0], split[1], split[2]);

		throw new IllegalStateException("malformed metadata reference: " + encoded);

	}

}

package eu.dnetlib.functionality.index.utils;

import java.io.StringReader;

import java.util.List;

import java.util.Map;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Autowired;

import org.xml.sax.InputSource;

import com.google.common.collect.HashBasedTable;

import com.google.common.collect.Table;

import com.mycila.xmltool.CallBack;

import com.mycila.xmltool.XMLDoc;

import com.mycila.xmltool.XMLTag;

import eu.dnetlib.data.provision.index.rmi.IndexServiceException;

import eu.dnetlib.functionality.index.client.IndexClientException;

public class MDFormatReader {

	static class XmlUtils {

		/**

		 * helper method, parses a list of fields.

		 *

		 * @param fields

		 *            the given fields

		 * @return the parsed fields

		 * @throws IndexServiceException

		 *             if cannot parse the fields

		 */

		public static Document parse(final String xml) {

			try {

				return new SAXReader().read(new StringReader(xml));

			} catch (DocumentException e) {

				throw new IllegalArgumentException("cannot parse: " + xml);

			}

		}

		public static InputSource asInputSource(final String input) throws DocumentException {

			return new InputSource(new StringReader(parse(input).asXML()));

		}

	}

	@Autowired

	private ServiceTools serviceTools;

	@Autowired

	private MetadataReferenceFactory mdFactory;

	public List<MetadataReference> listMDRefs() throws IndexClientException {

		return serviceTools.listMDRefs();

	}

	public Document getFields(final MetadataReference mdRef) {

		String fields = serviceTools.getIndexFields(mdRef);

		return (fields != null) && !fields.isEmpty() ? XmlUtils.parse(fields) : null;

	}

	public Map<String, String> getAttributeMap(final MetadataReference mdRef, final String attribute) throws IndexClientException {

		return getAttributeTable(mdRef, attribute).column(attribute);

	}

	public Table<String, String, String> getAttributeTable(final MetadataReference mdRef, final String... attributeList) throws IndexClientException {

		final String fields = serviceTools.getIndexFields(mdRef);

		if (fields.isEmpty()) throw new IndexClientException("No result getting index layout informations");

		final Table<String, String, String> t = HashBasedTable.create();

		XMLDoc.from(serviceTools.getIndexFields(mdRef), false).forEach("//FIELD", new CallBack() {

			@Override

			public void execute(final XMLTag field) {

				for (String attribute : attributeList) {

					String value = null;

					if ("xpath".equals(attribute)) {

						if (!(field.hasAttribute("xpath") || field.hasAttribute("value"))) return;

						value = field.hasAttribute("xpath") ? field.getAttribute("xpath") : field.getAttribute("value");

					}

					if ("weight".equals(attribute)) {

						if (!(field.hasAttribute(attribute))) return;

						value = field.hasAttribute(attribute) ? field.getAttribute(attribute) : "";

					}

					if (value == null) {

						value = field.getAttribute(attribute);

					}

					t.put(field.getAttribute("name").toLowerCase(), attribute, value);

				}

			}

		});

		return t;

	}

}

package eu.dnetlib.functionality.index.model;

import java.util.Date;

import java.util.List;

/**

 * Interface for a sequence of Any objects.

 * 

 * <p>

 * AnySeq does <b>not</b> allow <code>null</code> values!

 * </p>

 * 

 */

public interface AnySeq extends List<Any>, Any {

	/**

	 * @param element

	 *            The string object to add

	 * @return true if successfully added, false else

	 */

	boolean add(String element);

	/**

	 * Long, Integer, Short and Byte values will be converted to Value object of type LONG, all others to Value object

	 * of type DOUBLE.

	 * 

	 * @param number

	 *            The number object to add

	 * @return true if successfully added, false else

	 */

	boolean add(Number number);

	/**

	 * @param index

	 *            The index where to add the any object

	 * @param element

	 *            The any object to add

	 */

	@Override

	void add(int index, Any element);

	/**

	 * @param index

	 *            The index of the object to return

	 * @return The AnyMap matching to the index, an InvalidValueTypeException is thrown if the value is not of type

	 *         AnyMap

	 */

	AnyMap getMap(int index);

	/**

	 * @param index

	 *            The index of the object to return

	 * @return The AnySeq matching to this index, an InvalidValueTypeException is thrown if the value is not of type

	 */

	AnySeq getSeq(int index);

	/**

	 * @param index

	 *            The index of the object to return

	 * @return The value matching to this index, an InvalidValueTypeException is thrown if the value is no value type.

	 */

	Value getValue(int index);

	/**

	 * @param index

	 *            The index of the object to return

	 * @return The string value matching to this index, an InvalidValueTypeException is thrown if the value is not of

	 *         type string

	 */

	String getStringValue(int index);

	/**

	 * @param index

	 *            The index of the object to return

	 * @return The double value matching to this index, an InvalidValueTypeException is thrown if the value is not of

	 *         type double

	 */

	Double getDoubleValue(int index);

	/**

	 * @param index

	 *            The index of the object to return

	 * @return The long value matching to this index, an InvalidValueTypeException is thrown if the value is not of type

	 *         long

	 */

	Long getLongValue(int index);

	/**

	 * @param index

	 *            The index of the object to return

	 * @return The boolean value matching to this index, an InvalidValueTypeException is thrown if the value is not of

	 *         type boolean

	 */

	Boolean getBooleanValue(int index);

	/**

	 * @param index

	 *            The index of the object to return

	 * @return The date value matching to this index, an InvalidValueTypeException is thrown if the value is not of type

	 *         date

	 */

	Date getDateValue(int index);

	/**

	 * @param index

	 *            The index of the object to return

	 * @return The date time value matching to this index, an InvalidValueTypeException is thrown if the value is not of

	 *         type date time

	 */

	Date getDateTimeValue(int index);

	/**

	 * returns all values as a List of Strings.

	 * 

	 * @throws InvalidValueTypeException

	 *             if not all contained values are strings.

	 */

	List<String> asStrings();

	/**

	 * returns all values as a List of Long.

	 * 

	 * @throws InvalidValueTypeException

	 *             if not all contained values are Longs.