D-Net

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:context="http://www.springframework.org/schema/context"

	xmlns:p="http://www.springframework.org/schema/p"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd

		http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">

	<bean id="resultSetDescriptorPrototype" class="eu.dnetlib.enabling.resultset.push.ResultSetDescriptor"

		p:rangeLength="200" scope="prototype" />

	<bean id="resultSetDescriptorFactory" class="eu.dnetlib.springutils.beans.factory.PrototypeFactory">

		<lookup-method name="newInstance" bean="resultSetDescriptorPrototype" />

	</bean>

</beans>

<xsl:stylesheet version="1.0"

                xmlns:xsl="http://www.w3.org/1999/XSL/Transform">

	<xsl:template match="first">

		<second><xsl:value-of select="."/></second>

	</xsl:template>

</xsl:stylesheet>

org.apache.cxf.Logger=org.apache.cxf.common.logging.Log4jLogger

log4j.rootLogger=WARN, CONSOLE

log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender

log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout

log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n

log4j.logger.eu.dnetlib=INFO

log4j.logger.eu.dnetlib.enabling.is.sn=INFO

#log4j.logger.org.apache.cxf.interceptor=FATAL

log4j.logger.org.apache.cxf.ws.addressing.ContextUtils=FATAL

log4j.logger.eu.dnetlib.enabling.tools.AbstractBaseService=INFO

log4j.logger.eu.dnetlib.enabling.inspector=DEBUG

log4j.logger.eu.dnetlib.xml.database.LoggingTrigger=FATAL

#log4j.logger.eu.dnetlib.enabling.tools.registration=DEBUG

log4j.logger.eu.dnetlib.enabling.tools.registration.ServiceRegistrator=INFO

log4j.logger.eu.dnetlib.enabling.inspector=FATAL

log4j.logger.eu.dnetlib.enabling.resultset=DEBUG

log4j.logger.eu.dnetlib.enabling.inspector.SubscriptionController=DEBUG

log4j.logger.eu.dnetlib.springutils.stringtemplate.StringTemplateViewResolver=FATAL

log4j.logger.eu.dnetlib.enabling.is.sn.SynchronousNotificationSenderImpl=WARN

log4j.logger.eu.dnetlib.enabling.manager.msro.wf.RefreshIndexJob=DEBUG

log4j.logger.eu.dnetlib.enabling.ui=DEBUG

log4j.logger.eu.dnetlib.enabling.manager.msro.wf.RefreshIndexJob=DEBUG

log4j.logger.org.apache.zookeeper.ClientCnxn=FATAL

log4j.logger.eu.dnetlib.enabling.resultset.push=DEBUG

package eu.dnetlib.enabling.hcm.sn;

public class SubscriptionException extends Exception {

	/**

	 *

	 */

	private static final long serialVersionUID = 3281648062309941192L;

	public SubscriptionException() {

		super();

	}

	public SubscriptionException(final String message, final Throwable cause) {

		super(message, cause);

	}

	public SubscriptionException(final String message) {

		super(message);

	}

}

package eu.dnetlib.enabling.hcm.sn;

/**

 * This component takes care of subscribing the MSRO service to interesting events.

 *

 * @author marko

 *

 */

public interface HCMSubscriber {

	void subscribeAll() throws SubscriptionException;

}

package eu.dnetlib.enabling.hcm.sn;

import java.util.List;

import javax.annotation.PostConstruct;

import eu.dnetlib.enabling.actions.SubscriptionAction;

import eu.dnetlib.enabling.hcm.HostingContextManagerServiceImpl;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

/**

 * This component takes care of subscribing the SubscriptionAction(s) to interesting events.

 *

 * @author claudio

 *

 */

public class HCMSubscriberImpl implements HCMSubscriber {

	/**

	 * logger.

	 */

	private static final Log log = LogFactory.getLog(HCMSubscriberImpl.class);

	/**

	 * service locator.

	 */

	@Autowired

	private UniqueServiceLocator serviceLocator;

	/**

	 * hcm

	 */

	private HostingContextManagerServiceImpl hcm;

	/**

	 * subscription actions.

	 */

	@Autowired(required = false)

	private List<SubscriptionAction> actions;

	@PostConstruct

	public void printList() {

		log.info(getActions());

	}

	@Override

	public void subscribeAll() throws SubscriptionException {

		// final String profileId = this.hcm.getProfileId();

		// if (getActions() != null) {

		// for (final SubscriptionAction action : getActions()) {

		// log.info("dynamically subscribing to " + action.getTopicExpression());

		// this.serviceLocator.getService(ISSNService.class).subscribe(profileId, action.getTopicExpression(), 0);

		// }

		// }

	}

	public List<SubscriptionAction> getActions() {

		return this.actions;

	}

	public void setActions(final List<SubscriptionAction> actions) {

		this.actions = actions;

	}

	public HostingContextManagerServiceImpl getHcm() {

		return this.hcm;

	}

	@Required

	public void setHcm(final HostingContextManagerServiceImpl hcm) {

		this.hcm = hcm;

	}

	public UniqueServiceLocator getServiceLocator() {

		return this.serviceLocator;

	}

	public void setServiceLocator(final UniqueServiceLocator serviceLocator) {

		this.serviceLocator = serviceLocator;

	}

}

package eu.dnetlib.enabling.hcm;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import eu.dnetlib.enabling.tools.AbstractBaseService;

import eu.dnetlib.enabling.tools.blackboard.NotificationHandler;

import eu.dnetlib.rmi.enabling.HostingContextManagerService;

/**

 * CNR HostingContextManagerService implementation. Will conflict with NKUA! yes this is ok.

 *

 * @author marko

 *

 */

public class HostingContextManagerServiceImpl extends AbstractBaseService implements HostingContextManagerService {

	/**

	 * logger.

	 */

	private static final Log log = LogFactory.getLog(HostingContextManagerServiceImpl.class); // NOPMD by marko on 11/24/08 5:02 PM

	/**

	 * notification handler.

	 */

	private NotificationHandler notificationHandler; // NOPMD

	@Override

	public String echo(final String s) {

		return s;

	}

	@Override

	public void notify(final String subscrId, final String topic, final String isId, final String message) {

		// if (log.isDebugEnabled()) {

		// log.debug("---- service got notification ----");

		// log.debug("subscrId: " + subscrId);

		// log.debug("topic " + topic);

		// log.debug("isId " + isId);

		// log.debug("msg: " + message);

		// log.debug("____ now processing the notification ____");

		// }

		// getNotificationHandler().notified(subscrId, topic, isId, message);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.enabling.tools.AbstractBaseService#start()

	 */

	@Override

	public void start() {

		log.info("staring hosting context manager");

	}

	public NotificationHandler getNotificationHandler() {

		return this.notificationHandler;

	}

	// @Required

	public void setNotificationHandler(final NotificationHandler notificationHandler) {

		this.notificationHandler = notificationHandler;

	}

}

package eu.dnetlib.enabling.resultset.listener;

import eu.dnetlib.rmi.common.ResultSetException;

/**

 * @author michele

 *

 */

public interface ResultSetListener<T> {

	boolean hasNext();

	T next() throws ResultSetException;

	int getCount();

	int getTotal();

}

package eu.dnetlib.enabling.resultset.listener;

import java.util.Collection;

import java.util.Iterator;

import eu.dnetlib.rmi.common.ResultSetException;

public class IterableResultSetListener<T> implements ResultSetListener<T> {

	private Iterator<T> iter;

	private int count;

	private int total;

	public IterableResultSetListener(final Iterable<T> iter) {

		this(iter.iterator());

		if (iter instanceof Collection) {

			this.total = ((Collection<?>) iter).size();

		}

	}

	public IterableResultSetListener(final Iterator<T> iter) {

		this.iter = iter;

		this.count = 0;

		this.total = -1;

	}

	@Override

	public boolean hasNext() {

		return this.iter.hasNext();

	}

	@Override

	public T next() throws ResultSetException {

		final T res = this.iter.next();

		if (res != null) {

			this.count++;

			return res;

		}

		throw new ResultSetException("Found a null element in resultset");

	}

	@Override

	public int getCount() {

		return this.count;

	}

	@Override

	public int getTotal() {

		return hasNext() ? this.total : this.count;

	}

}

package eu.dnetlib.enabling.resultset;

import javax.xml.bind.annotation.XmlRootElement;

import eu.dnetlib.enabling.resultset.listener.ResultSetListener;

@XmlRootElement

public class ResultSetInfo {

	private String id;

	private int cursor = -1;

	private int total = -1;

	private boolean inaccurate = false;

	public ResultSetInfo() {}

	public ResultSetInfo(final String id, final ResultSetListener<?> listener) {

		this.id = id;

		this.cursor = listener.getCount();

		this.total = listener.getTotal();

		this.inaccurate = listener.getCount() < 0 || listener.getTotal() < 0 || listener.getTotal() < listener.getCount();

	}

	public ResultSetInfo(final String id, final int cursor, final int total, final boolean inaccurate) {

		this.id = id;

		this.cursor = cursor;

		this.total = total;

		this.inaccurate = inaccurate;

	}

	public String getId() {

		return this.id;

	}

	public void setId(final String id) {

		this.id = id;

	}

	public int getTotal() {

		return this.total;

	}

	public void setTotal(final int total) {

		this.total = total;

	}

	public int getCursor() {

		return this.cursor;

	}

	public void setCursor(final int cursor) {

		this.cursor = cursor;

	}

	public boolean isInaccurate() {

		return this.inaccurate;

	}

	public void setInaccurate(final boolean inaccurate) {

		this.inaccurate = inaccurate;

	}

}

package eu.dnetlib.enabling.resultset.factory;

import java.util.Map;

import java.util.function.Function;

import java.util.stream.StreamSupport;

import javax.xml.transform.TransformerFactory;

import javax.xml.transform.dom.DOMSource;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

import org.springframework.core.io.Resource;

import eu.dnetlib.enabling.resultset.client.ResultSetClient;

import eu.dnetlib.enabling.resultset.listener.IterableResultSetListener;

import eu.dnetlib.enabling.resultset.listener.ResultSetListener;

import eu.dnetlib.enabling.resultset.registry.ResultSetRegistry;

import eu.dnetlib.miscutils.functional.xml.ApplyXslt;

import eu.dnetlib.rmi.common.ResultSet;

public class ResultSetFactory {

	private ResultSetRegistry resultSetRegistry;

	private ResultSetClient resultSetClient;

	private String baseUrl;

	@Autowired

	private TransformerFactory saxonTransformerFactory;

	public <T> ResultSet<T> registerResultSet(final ResultSetListener<T> rs) {

		final String rsId = this.resultSetRegistry.registerResultSet(rs);

		return new ResultSet<T>(rsId, this.baseUrl);

	}

	public <T> ResultSet<T> createResultSet(final ResultSetListener<T> listener) {

		return registerResultSet(listener);

	}

	public <T> ResultSet<T> createResultSet(final Iterable<T> iter) {

		return registerResultSet(new IterableResultSetListener<T>(iter));

	}

	public <T, K> ResultSet<K> map(final ResultSet<?> rsIn, final Class<T> clazzIn, final Function<T, K> mapper) {

		final Iterable<T> iterIn = this.resultSetClient.iter(rsIn, clazzIn);

		return createResultSet(() -> StreamSupport.stream(iterIn.spliterator(), false).map(mapper).iterator());

	}

	public ResultSet<String> xsltMap(final ResultSet<?> rsIn, final String xslt) {

		return map(rsIn, String.class, new ApplyXslt(xslt, saxonTransformerFactory));

	}

	public ResultSet<String> xsltMap(final ResultSet<?> rsIn, final String xslt, final Map<String, String> params) {

		return map(rsIn, String.class, new ApplyXslt(xslt, null, params, saxonTransformerFactory));

	}

	public ResultSet<String> xsltMap(final ResultSet<?> rsIn, final Resource xsltResource) {

		return map(rsIn, String.class, new ApplyXslt(xsltResource,saxonTransformerFactory));

	}

	public ResultSet<String> xsltMap(final ResultSet<?> rsIn, final Resource xsltResource, final Map<String, String> params) {

		return map(rsIn, String.class, new ApplyXslt(xsltResource, params, saxonTransformerFactory));

	}

	public ResultSet<String> xsltMap(final ResultSet<?> rsIn, final DOMSource domSource) {

		return map(rsIn, String.class, new ApplyXslt(domSource,saxonTransformerFactory));

	}

	public ResultSetRegistry getResultSetRegistry() {

		return this.resultSetRegistry;

	}

	@Required

	public void setResultSetRegistry(final ResultSetRegistry resultSetRegistry) {

		this.resultSetRegistry = resultSetRegistry;

	}

	public String getBaseUrl() {

		return this.baseUrl;

	}

	@Required

	public void setBaseUrl(final String baseUrl) {

		this.baseUrl = baseUrl;

	}

	public ResultSetClient getResultSetClient() {

		return this.resultSetClient;

	}

	@Required

	public void setResultSetClient(final ResultSetClient resultSetClient) {

		this.resultSetClient = resultSetClient;

	}

}

package eu.dnetlib.enabling.resultset.registry;

import eu.dnetlib.enabling.resultset.listener.ResultSetListener;

import eu.dnetlib.rmi.common.ResultSetException;

/**

 * Instances of ResultSetRegistry manage a set of resultset objects and manage their garbage collection.

 *

 * @author michele

 *

 */

public interface ResultSetRegistry {

	/**

	 * register a resultSetListener

	 *

	 * @param listener

	 *            a resultsetListener object

	 * @return the resultset id

	 */

	String registerResultSet(ResultSetListener<?> listener);

	/**

	 * obtain the resultsetListener with the given id.

	 *

	 * @param rsId

	 *            resultset id

	 * @return the resultsetListener object matching the rsId or null

	 * @throws ResultSetException

	 */

	<T> ResultSetListener<T> getResultSetById(String rsId) throws ResultSetException;

	/**

	 *

	 * @param id

	 * @return

	 */

	boolean contains(String id);

}

package eu.dnetlib.enabling.resultset.registry;

import java.util.HashMap;

import java.util.Map;

import java.util.Set;

import java.util.UUID;

import java.util.stream.Collectors;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.enabling.resultset.listener.ResultSetListener;

import eu.dnetlib.miscutils.datetime.DateUtils;

import eu.dnetlib.rmi.common.ResultSetException;

public class ResultSetRegistryImpl implements ResultSetRegistry {

	private static final String RS_PREFIX = "rs-";

	private long ttl = 600000;

	private static final Log log = LogFactory.getLog(ResultSetRegistry.class);

	private class RsCacheElem {

		private final ResultSetListener<?> listener;

		private long lastDate = DateUtils.now();

		public RsCacheElem(final ResultSetListener<?> listener) {

			this.listener = listener;

		}

		public ResultSetListener<?> getListener() {

			this.lastDate = DateUtils.now();

			return this.listener;

		}

		public boolean isOld(final long now) {

			return (now - this.lastDate) > ResultSetRegistryImpl.this.ttl;

		}

	}

	private final Map<String, RsCacheElem> cache = new HashMap<>();

	@Override

	public String registerResultSet(final ResultSetListener<?> listener) {

		verifyCache();

		final String newId = RS_PREFIX + UUID.randomUUID();

		this.cache.put(newId, new RsCacheElem(listener));

		log.debug("New resultset created: " + newId);

		return newId;

	}

	private void verifyCache() {

		final long now = DateUtils.now();

		final Set<String> toDelete = this.cache.entrySet()

				.stream()

				.filter(e -> e.getValue().isOld(now))

				.map(e -> e.getKey())

				.collect(Collectors.toSet());

		if (!toDelete.isEmpty()) {

			log.debug("The following resultsets are expired: " + toDelete);

		}

		toDelete.forEach(this.cache::remove);

	}

	@SuppressWarnings("unchecked")

	@Override

	public <T> ResultSetListener<T> getResultSetById(final String rsId) throws ResultSetException {

		final RsCacheElem elem = this.cache.get(rsId);

		if (elem != null) {

			final ResultSetListener<T> rs = (ResultSetListener<T>) elem.getListener();

			if (rs != null) { return rs; }

		}

		throw new ResultSetException("Missing resultset with id " + rsId);

	}

	@Override

	public boolean contains(final String rsId) {

		return this.cache.get(rsId) != null;

	}

	public long getTtl() {

		return this.ttl;

	}

	@Required

	public void setTtl(final long ttl) {

		this.ttl = ttl;

	}

}

package eu.dnetlib.enabling.resultset;

import java.util.ArrayList;

import java.util.List;

import javax.xml.bind.annotation.XmlRootElement;

@XmlRootElement

public class ResultSetResponse {

	private int from = 0;

	private int to = 0;

	private boolean end = false;

	private int total = -1;

	private List<String> elements = new ArrayList<>();

	public ResultSetResponse() {}

	public ResultSetResponse(final int from, final int to, final boolean end, final int total, final List<String> elements) {

		this.from = from;

		this.to = to;

		this.end = end;

		this.total = total;

		this.elements = elements;

	}

	public int getFrom() {

		return this.from;

	}

	public void setFrom(final int from) {

		this.from = from;

	}

	public int getTo() {

		return this.to;

	}

	public void setTo(final int to) {

		this.to = to;

	}

	public boolean isEnd() {

		return this.end;

	}

	public void setEnd(final boolean end) {

		this.end = end;

	}

	public int getTotal() {

		return this.total;

	}

	public void setTotal(final int total) {

		this.total = total;

	}

	public List<String> getElements() {

		return this.elements;

	}

	public void setElements(final List<String> elements) {

		this.elements = elements;

	}

}

package eu.dnetlib.enabling.resultset;

import java.util.ArrayList;

import java.util.List;

import com.google.gson.Gson;

import eu.dnetlib.enabling.resultset.listener.ResultSetListener;

import eu.dnetlib.enabling.resultset.registry.ResultSetRegistry;

import eu.dnetlib.rmi.common.ResultSetException;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.stereotype.Controller;

import org.springframework.web.bind.annotation.PathVariable;

import org.springframework.web.bind.annotation.RequestMapping;

import org.springframework.web.bind.annotation.RequestMethod;

import org.springframework.web.bind.annotation.ResponseBody;

@Controller

public class ResultSetController {

	@Autowired

	private ResultSetRegistry resultSetRegistry;

	@RequestMapping(value = "/rs/{rsId}/next/{n}", method = RequestMethod.GET)

	public @ResponseBody ResultSetResponse getNextResults(@PathVariable(value = "rsId") final String rsId, @PathVariable(value = "n") final int n)

			throws ResultSetException {

		final ResultSetListener<?> rs = this.resultSetRegistry.getResultSetById(rsId);

		final int start = rs.getCount();

		final List<String> elements = new ArrayList<>();

		final Gson gson = new Gson();

		while (rs.hasNext() && elements.size() < n) {

			elements.add(gson.toJson(rs.next()));

		}

		final int from = start;

		final int to = rs.getCount();

		final boolean end = !rs.hasNext();

		final int total = rs.getTotal();

		return new ResultSetResponse(from, to, end, total, elements);

	}

	@RequestMapping(value = "/rs/{rsId}/info", method = RequestMethod.GET)

	public @ResponseBody ResultSetInfo getRsInfo(@PathVariable(value = "rsId") final String rsId)

			throws ResultSetException {

		final ResultSetListener<?> rs = this.resultSetRegistry.getResultSetById(rsId);

		return new ResultSetInfo(rsId, rs);

	}

}

package eu.dnetlib.enabling.resultset.client;

import java.io.InputStream;

import java.util.List;

import java.util.stream.Collectors;

import com.google.gson.Gson;

import eu.dnetlib.enabling.resultset.ResultSetResponse;

import eu.dnetlib.rmi.common.ResultSetException;

import org.apache.commons.io.IOUtils;

import org.apache.http.HttpStatus;

import org.apache.http.client.methods.CloseableHttpResponse;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.impl.client.CloseableHttpClient;

import org.apache.http.impl.client.HttpClients;

public class HttpResultSetClientIterator<T> extends AbstractResultSetClientIterator<T> {

	private static final int PAGE_SIZE = 20;

	private final String id;

	private final String baseUrl;

	private final Class<T> clazz;

	private final CloseableHttpClient client = HttpClients.createDefault();

	public HttpResultSetClientIterator(final String id, final String baseUrl, final Class<T> clazz) {

		super();

		this.id = id;

		this.baseUrl = baseUrl;

		this.clazz = clazz;

	}

	@Override

	protected List<T> nextPage() throws ResultSetException {

		final HttpGet method = new HttpGet(getBaseUrl() + "/" + getId() + "/next/" + PAGE_SIZE);

		try(CloseableHttpResponse httpResponse = client.execute(method)) {

			int statusCode = httpResponse.getStatusLine().getStatusCode();

			if (HttpStatus.SC_OK != statusCode) {

				throw new ResultSetException("Error " + statusCode + " dowloading url: " + getBaseUrl());

			}

			try(InputStream responseBody = httpResponse.getEntity().getContent()) {

				final String json = IOUtils.toString(responseBody);

				final Gson gson = new Gson();

				final ResultSetResponse response = gson.fromJson(json, ResultSetResponse.class);

				return response.getElements()

						.stream()

						.map(s -> gson.fromJson(s, getClazz()))

						.collect(Collectors.toList());

			}

		} catch (Throwable e) {

			throw new ResultSetException("Error dowloading url: " + getBaseUrl(), e);

		}

	}

	public String getId() {

		return this.id;

	}

	public String getBaseUrl() {

		return this.baseUrl;

	}

	public Class<T> getClazz() {

		return this.clazz;

	}

}

package eu.dnetlib.enabling.resultset.client;

import java.util.Iterator;

import java.util.LinkedList;

import java.util.List;

import java.util.NoSuchElementException;

import java.util.Queue;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import eu.dnetlib.rmi.common.ResultSetException;

public abstract class AbstractResultSetClientIterator<T> implements Iterator<T> {

	private static final Log log = LogFactory.getLog(AbstractResultSetClientIterator.class);

	private final Queue<T> buffer = new LinkedList<T>();

	@Override

	public boolean hasNext() {

		if (!this.buffer.isEmpty()) { return true; }

		try {

			return refillBuffer();

		} catch (final ResultSetException e) {

			log.error("Error refilling resultSet buffer", e);

			throw new RuntimeException("Error refilling resultSet buffer", e);

		}

	}

	@Override

	public T next() {

		if (!hasNext()) {

			log.error("NoSuchElementException");

			throw new NoSuchElementException();

		}

		return this.buffer.poll();

	}

	@Override

	public void remove() {

		throw new RuntimeException("Not implemented");

	}

	private boolean refillBuffer() throws ResultSetException {

		final List<T> page = nextPage();

		if (page == null || page.isEmpty()) { return false; }

		this.buffer.addAll(page);

		return true;

	}

	abstract protected List<T> nextPage() throws ResultSetException;

}

package eu.dnetlib.enabling.resultset.client;

import java.io.IOException;

import java.io.InputStream;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.http.HttpStatus;

import org.apache.http.client.methods.CloseableHttpResponse;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.impl.client.CloseableHttpClient;

import org.apache.http.impl.client.HttpClients;

import org.springframework.beans.factory.annotation.Required;

import com.google.gson.Gson;

import eu.dnetlib.enabling.resultset.ResultSetInfo;

import eu.dnetlib.enabling.resultset.listener.ResultSetListener;

import eu.dnetlib.enabling.resultset.registry.ResultSetRegistry;

import eu.dnetlib.rmi.common.ResultSet;

import eu.dnetlib.rmi.common.ResultSetException;

public class ResultSetClient {

	private ResultSetRegistry resultSetRegistry;

	private static final Log log = LogFactory.getLog(ResultSetClient.class);

	public <T> Iterable<T> iter(final ResultSet<?> resultSet, final Class<T> clazz) {

		try {

			if (this.resultSetRegistry.contains(resultSet.getId())) {

				final ResultSetListener<?> listener = this.resultSetRegistry.getResultSetById(resultSet.getId());

				return () -> new LocalResultSetClientIterator<>(listener, clazz);

			} else {

				return () -> new HttpResultSetClientIterator<>(resultSet.getId(), resultSet.getBaseUrl(), clazz);

			}

		} catch (final Throwable e) {

			log.error("Error accessing resultset: " + resultSet.getId());

			throw new RuntimeException(e);

		}

	}

	public ResultSetInfo info(final ResultSet<?> resultSet) {

		try {

			if (this.resultSetRegistry.contains(resultSet.getId())) {

				final ResultSetListener<?> listener = this.resultSetRegistry.getResultSetById(resultSet.getId());

				return new ResultSetInfo(resultSet.getId(), listener);

			} else {

				return invokeRemoteInfo(resultSet);

			}

		} catch (final Throwable e) {

			log.error("Error accessing resultset: " + resultSet.getId());

			throw new RuntimeException(e);

		}

	}

	private ResultSetInfo invokeRemoteInfo(final ResultSet<?> rs) throws ResultSetException {

		final HttpGet method = new HttpGet(rs.getBaseUrl() + "/" + rs.getId() + "/info");

		try(CloseableHttpClient httpClient = HttpClients.createDefault()) {

			try(CloseableHttpResponse response = httpClient.execute(method)) {

				int statusCode = response.getStatusLine().getStatusCode();

				if (HttpStatus.SC_OK != statusCode) {

					throw new ResultSetException("Error " + statusCode + " dowloading url: " + rs.getBaseUrl());

				}

				try(InputStream responseBody = response.getEntity().getContent()) {

					String json = IOUtils.toString(responseBody);

					return new Gson().fromJson(json, ResultSetInfo.class);

				}

			} catch (final Throwable e) {

				throw new ResultSetException("Error dowloading url: " + rs.getBaseUrl(), e);

			}

		} catch (IOException e) {

			log.error("Can't close connections gracefully", e);

			throw new ResultSetException("Can't close connections gracefully", e);

		}

	}

	public ResultSetRegistry getResultSetRegistry() {

		return this.resultSetRegistry;

	}

	@Required

	public void setResultSetRegistry(final ResultSetRegistry resultSetRegistry) {

		this.resultSetRegistry = resultSetRegistry;

	}

}

package eu.dnetlib.enabling.resultset.client;

import java.util.ArrayList;

import java.util.List;

import eu.dnetlib.enabling.resultset.listener.ResultSetListener;

import eu.dnetlib.rmi.common.ResultSetException;

public class LocalResultSetClientIterator<T> extends AbstractResultSetClientIterator<T> {

	private static final int PAGE_SIZE = 20;

	private final ResultSetListener<?> listener;

	private final Class<T> clazz;

	public LocalResultSetClientIterator(final ResultSetListener<?> listener, final Class<T> clazz) {

		super();

		this.listener = listener;

		this.clazz = clazz;

	}

	@Override

	protected List<T> nextPage() throws ResultSetException {

		final List<T> res = new ArrayList<>();

		for (int i = 0; i < PAGE_SIZE && this.listener.hasNext(); i++) {

			res.add(this.clazz.cast(this.listener.next()));

		}

		return res;

	}

}

package eu.dnetlib.enabling.datasources;

import java.util.Map.Entry;

import eu.dnetlib.rmi.datasource.IfaceDesc;

import org.dom4j.DocumentHelper;

import org.dom4j.Element;

public class IfaceDescToNode {

	public static Element convert(final IfaceDesc iface) throws Exception {

		final Element ifcNode = DocumentHelper.createElement("INTERFACE");

		ifcNode.addAttribute("id", iface.getId());

		ifcNode.addAttribute("label", iface.getTypology() + " (" + iface.getCompliance() + ")");

		ifcNode.addAttribute("compliance", iface.getCompliance());

		ifcNode.addAttribute("typology", iface.getTypology());

		ifcNode.addAttribute("contentDescription", iface.getContentDescription());

		ifcNode.addAttribute("active", Boolean.toString(iface.getActive()));

		ifcNode.addAttribute("removable", Boolean.toString(iface.getRemovable()));

		final Element acProtoNode = ifcNode.addElement("ACCESS_PROTOCOL");

		for (Entry<String, String> e : iface.getAccessParams().entrySet()) {

			acProtoNode.addAttribute(e.getKey(), e.getValue());

		}

		acProtoNode.setText(iface.getAccessProtocol());

		ifcNode.addElement("BASE_URL").setText(iface.getBaseUrl());

		;

		for (Entry<String, String> e : iface.getExtraFields().entrySet()) {

			final Element field = ifcNode.addElement("INTERFACE_EXTRA_FIELD");

			field.addAttribute("name", e.getKey());

			field.setText(e.getValue());

		}

		return ifcNode;

	}

}

package eu.dnetlib.enabling.datasources;

import eu.dnetlib.rmi.datasource.BrowsableField;

public class XmlBrowsableField extends BrowsableField {

	private String xpath;

	private String vocabulary;

	public XmlBrowsableField() {

		super();

	}

	public XmlBrowsableField(final String id, final String label) {

		super(id, label);

	}

	public XmlBrowsableField(final String id, final String label, final String xpath, final String vocabulary) {

		super(id, label);

		this.xpath = xpath;

		this.vocabulary = vocabulary;

	}

	public String getXpath() {

		return xpath;

	}

	public void setXpath(final String xpath) {

		this.xpath = xpath;

	}

	public String getVocabulary() {

		return vocabulary;

	}

	public void setVocabulary(final String vocabulary) {

		this.vocabulary = vocabulary;

	}

}

package eu.dnetlib.enabling.datasources;

import java.util.Date;

import java.util.List;

import java.util.Map;

import eu.dnetlib.enabling.tools.AbstractBaseService;

import eu.dnetlib.rmi.datasource.*;

import org.springframework.beans.factory.annotation.Required;

public class DatasourceManagerServiceImpl extends AbstractBaseService implements DatasourceManagerService {

	//private List<XmlBrowsableField> browsableFields;

	private DnetDatasourceManagerCore datasourceManagerCore;

	@Override

	public boolean addDatasource(final DatasourceDesc ds) throws DatasourceManagerServiceException {

		return datasourceManagerCore.addDatasource(ds);

	}

	@Override

	public boolean deleteDatasource(final String dsId) throws DatasourceManagerServiceException {

		return datasourceManagerCore.deleteDatasource(dsId);

	}

	@Override

	public DatasourceDesc getDatasource(final String dsId) throws DatasourceManagerServiceException {

		return datasourceManagerCore.getDatasource(dsId);

	}

	@Override

	public List<DatasourceDesc> listAllDatasources() throws DatasourceManagerServiceException {

		return listDatasourcesUsingFilter(null, null, null, null);

	}

	@Override

	public List<DatasourceDesc> listDatasourcesUsingFilter(final String compliance,

			final String contentDescription,

			final String iisProcessingWorkflow,

			final String collectedFrom)

			throws DatasourceManagerServiceException {

		return datasourceManagerCore.listDatasourcesUsingFilter(compliance, contentDescription, iisProcessingWorkflow, collectedFrom);

	}

	@Override

	public boolean updateLevelOfCompliance(final String dsId, final String ifaceId, final String level) throws DatasourceManagerServiceException {

		return datasourceManagerCore.updateLevelOfCompliance(dsId, ifaceId, level);

	}

	@Override

	public boolean updateBaseUrl(final String dsId, final String ifaceId, final String baseUrl) throws DatasourceManagerServiceException {

		return datasourceManagerCore.updateBaseUrl(dsId, ifaceId, baseUrl);

	}

	@Override

	public boolean updateActivationStatus(final String dsId, final String ifaceId, final boolean active) throws DatasourceManagerServiceException {

		return datasourceManagerCore.updateActivationStatus(dsId, ifaceId, active);

	}

	@Override

	public boolean updateContentDescription(final String dsId, final String ifaceId, final String desc) throws DatasourceManagerServiceException {

		return datasourceManagerCore.updateContentDescription(dsId, ifaceId, desc);

	}

	@Override

	public boolean setIisProcessingWorkflow(final String dsId, final String ifaceId, final String wf) throws DatasourceManagerServiceException {

		return datasourceManagerCore.setIisProcessingWorkflow(dsId, ifaceId, wf);

	}

	@Override

	public boolean updateExtraField(final String dsId, final String ifaceId, final String field, final String value, final boolean preserveOriginal)

			throws DatasourceManagerServiceException {

		return datasourceManagerCore.updateExtraField(dsId, ifaceId, field, value, preserveOriginal);

	}

	@Override

	public boolean updateAccessParam(final String dsId, final String ifaceId, final String field, final String value, final boolean preserveOriginal)

			throws DatasourceManagerServiceException {

		return datasourceManagerCore.updateAccessParam(dsId, ifaceId, field, value, preserveOriginal);

	}

	@Override

	public boolean deleteAccessParamOrExtraField(final String dsId, final String ifaceId, final String field) throws DatasourceManagerServiceException {

		return datasourceManagerCore.deleteAccessParamOrExtraField(dsId, ifaceId, field);

	}

	@Override

	public boolean addInterface(final String dsId, final IfaceDesc iface) throws DatasourceManagerServiceException {

		return datasourceManagerCore.addInterface(dsId, iface);

	}

	@Override

	public boolean deleteInterface(final String dsId, final String ifaceId) throws DatasourceManagerServiceException {

		return datasourceManagerCore.deleteInterface(dsId, ifaceId);

	}

	@Override

	public boolean updateSQL(final String dsId, final String sql, final boolean delete) throws DatasourceManagerServiceException {

		return datasourceManagerCore.updateSQL(dsId, sql, delete);

	}

	@Override

	public Date findNextScheduledExecution(final String dsId, final String ifaceId) throws DatasourceManagerServiceException {

		return datasourceManagerCore.findNextScheduledExecution(dsId, ifaceId);

	}

	@Override

	public boolean bulkUpdateApiExtraFields(final String repoId, final String ifaceId, final Map<String, String> fields)

			throws DatasourceManagerServiceException {

		return datasourceManagerCore.bulkUpdateApiExtraFields(repoId, ifaceId, fields);

	}

	@Override

	public boolean bulkUpdateApiAccessParams(final String dsId, final String ifaceId, final Map<String, String> params)

			throws DatasourceManagerServiceException {

		return datasourceManagerCore.bulkUpdateApiAccessParams(dsId, ifaceId, params);

	}

	@Override

	public boolean overrideCompliance(final String dsId, final String ifaceId, final String level) throws DatasourceManagerServiceException {

		return datasourceManagerCore.overrideCompliance(dsId, ifaceId, level);

	}

	@Override

	public List<BrowsableField> listBrowsableFields() throws DatasourceManagerServiceException {

		return datasourceManagerCore.listBrowsableFields();

	}

	@Override

	public List<BrowseTerm> browseField(final String f) throws DatasourceManagerServiceException {

		return datasourceManagerCore.browseField(f);

	}

	@Override

	public List<SearchInterfacesEntry> searchInterface(final String field, final String value) throws DatasourceManagerServiceException {

		return datasourceManagerCore.searchInterface(field, value);

	}

	@Override

	public List<RepositoryMapEntry> getRepositoryMap() throws DatasourceManagerServiceException {

		return datasourceManagerCore.getRepositoryMap();

	}

	@Override

	public List<SimpleDatasourceDesc> simpleListDatasourcesByType(final String type) throws DatasourceManagerServiceException {

		return datasourceManagerCore.simpleListDatasourcesByType(type);

	}

	public DnetDatasourceManagerCore getDatasourceManagerCore() {

		return datasourceManagerCore;

	}

	@Required

	public void setDatasourceManagerCore(final DnetDatasourceManagerCore datasourceManagerCore) {

		this.datasourceManagerCore = datasourceManagerCore;

	}

}

package eu.dnetlib.enabling.datasources;

import java.util.Date;

import java.util.List;

import java.util.Map;

import eu.dnetlib.rmi.datasource.*;

/**

 * Created by sandro on 6/29/16.

 */

public interface DnetDatasourceManagerCore {

	boolean addDatasource(DatasourceDesc datasourceDescription) throws DatasourceManagerServiceException;

	boolean deleteDatasource(final String datasourceIdentifier) throws DatasourceManagerServiceException;

	DatasourceDesc getDatasource(final String datasourceIdentifier) throws DatasourceManagerServiceException;

	List<DatasourceDesc> listDatasourcesUsingFilter(final String compliance, final String contentDescription,

			final String iisProcessingWorkflow,

			final String collectedFrom) throws DatasourceManagerServiceException;

	boolean deleteInterface(final String dsId, final String ifaceId) throws DatasourceManagerServiceException;

	boolean updateLevelOfCompliance(final String dsId, final String interfaceId, final String level) throws DatasourceManagerServiceException;

	boolean updateBaseUrl(final String dsId, final String ifaceId, final String baseUrl) throws DatasourceManagerServiceException;

	boolean updateActivationStatus(final String dsId, final String ifaceId, final boolean active) throws DatasourceManagerServiceException;

	boolean updateContentDescription(final String dsId, final String ifaceId, final String desc) throws DatasourceManagerServiceException;

	boolean setIisProcessingWorkflow(final String dsId, final String ifaceId, final String wf) throws DatasourceManagerServiceException;

	boolean updateExtraField(final String dsId, final String ifaceId, final String field, final String value, final boolean preserveOriginal)

			throws DatasourceManagerServiceException;

	boolean updateAccessParam(final String dsId, final String ifaceId, final String field, final String value, final boolean preserveOriginal)

			throws DatasourceManagerServiceException;

	boolean deleteAccessParamOrExtraField(final String dsId, final String ifaceId, final String field) throws DatasourceManagerServiceException;

	boolean addInterface(final String dsId, final IfaceDesc iface) throws DatasourceManagerServiceException;

	List<DatasourceDesc> listAllDatasources() throws DatasourceManagerServiceException;

	boolean overrideCompliance(String dsId, String ifaceId, String level) throws DatasourceManagerServiceException;

	boolean updateSQL(String dsId, String sql, boolean delete) throws DatasourceManagerServiceException;

	Date findNextScheduledExecution(String dsId, String ifaceId) throws DatasourceManagerServiceException;

	boolean bulkUpdateApiExtraFields(String dsId, String ifaceId, Map<String, String> fields) throws DatasourceManagerServiceException;

	boolean bulkUpdateApiAccessParams(String dsId, String ifaceId, Map<String, String> params) throws DatasourceManagerServiceException;

	List<BrowsableField> listBrowsableFields() throws DatasourceManagerServiceException;

	List<BrowseTerm> browseField(final String field) throws DatasourceManagerServiceException;

	List<SearchInterfacesEntry> searchInterface(final String field, final String value)

			throws DatasourceManagerServiceException;

	List<RepositoryMapEntry> getRepositoryMap() throws DatasourceManagerServiceException;

	List<SimpleDatasourceDesc> simpleListDatasourcesByType(String type) throws DatasourceManagerServiceException;

}

package eu.dnetlib.enabling.datasources;

import java.io.StringReader;

import java.text.ParseException;

import java.util.*;

import java.util.stream.Collectors;

import com.google.common.collect.Sets;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.rmi.datasource.*;

import eu.dnetlib.rmi.enabling.*;

import org.antlr.stringtemplate.StringTemplate;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringEscapeUtils;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.lang3.math.NumberUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.Element;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import org.quartz.CronExpression;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

import org.springframework.core.io.ClassPathResource;

/**

 * Created by sandro on 6/29/16.

 */

public class DnetDatasourceManagerCoreImpl implements DnetDatasourceManagerCore {

	private static final Log log = LogFactory.getLog(DnetDatasourceManagerCoreImpl.class);

	private static final String REPOSITORY_RESOURCE_TYPE = "RepositoryServiceResourceType";

	private List<XmlBrowsableField> browsableFields;

	@Autowired

	private UniqueServiceLocator serviceLocator;

	private ClassPathResource findReposQueryTmpl = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/findRepos.xquery.st");

	private ClassPathResource browseRepoApisQueryTmpl = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/browseRepoApis.xquery.st");

	private ClassPathResource findRepoApisQueryTmpl = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/findRepoApis.xquery.st");

	private ClassPathResource findReposMapQuery = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/findReposMap.xquery");

	private ClassPathResource simpleFindReposQueryTmpl = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/simpleFindRepos.xquery.st");

	@Override

	public boolean addDatasource(final DatasourceDesc datasourceDescription) throws DatasourceManagerServiceException {

		try {

			final String profile = DatasourceDescToProfile.convert(datasourceDescription);

			serviceLocator.getService(ISRegistryService.class).registerProfile(profile);

			return true;

		} catch (Exception e) {

			log.error("Error registering profile", e);

			throw new DatasourceManagerServiceException("Error registering profile", e);

		}

	}

	@Override

	public boolean deleteDatasource(final String datasourceIdentifier) throws DatasourceManagerServiceException {

		return _deleteDatasource(fixDsId(datasourceIdentifier));

	}

	@Override

	public DatasourceDesc getDatasource(final String datasourceIdentifier) throws DatasourceManagerServiceException {

		return _getDatasource(fixDsId(datasourceIdentifier));

	}

	@Override

	public List<DatasourceDesc> listDatasourcesUsingFilter(final String compliance,

			final String contentDescription,

			final String iisProcessingWorkflow,

			final String collectedFrom) throws DatasourceManagerServiceException {

		try {

			final StringTemplate st = new StringTemplate();

			st.setTemplate(IOUtils.toString(findReposQueryTmpl.getInputStream()));

			final Map<String, String> conds = new HashMap<>();