D-Net

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>cnr-modular-mdstore-service</artifactId>

	<packaging>jar</packaging>

	<version>3.0.0</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/cnr-modular-mdstore-service/tags/cnr-modular-mdstore-service-3.0.0</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.springframework</groupId>

			<artifactId>spring-test</artifactId>

			<version>${spring.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.mockito</groupId>

			<artifactId>mockito-core</artifactId>

			<version>1.6</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-test-utils</artifactId>

			<version>[1.0.0,2.0.0)</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-rmi-api</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-blackboard-common</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-misc-utils</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-resultset-service</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-resultset-client</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

	</dependencies>

</project>

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws"

	xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing"

	xmlns:p="http://www.springframework.org/schema/p" xmlns:http="http://cxf.apache.org/transports/http/configuration"

	xmlns:t="http://dnetlib.eu/springbeans/t" xmlns:template="http://dnetlib.eu/springbeans/template"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd

                                    http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd

                                    http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd

                                    http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd

                            http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd

                            http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">

	<!-- beans -->

	<bean id="mdStoreService" class="eu.dnetlib.data.mdstore.modular.ModularMDStoreService"

		init-method="start" destroy-method="stop" p:notificationHandler-ref="mdstoreNotificationHandler"

		p:iterableResultSetFactory-ref="iterableResultSetFactory"

		p:feeder-ref="mdstoreFeeder" p:retriever-ref="mdstoreRetriever" />

	<bean id="mdstoreNotificationHandler"

		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerExecutorNotificationHandler"

		p:blackboardExecutor-ref="mdstoreBlackboardExecutor" />

	<bean id="mdstoreBlackboardExecutor"

		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerActionExecutor"

		p:blackboardHandler-ref="blackboardHandler"

		p:actionType="eu.dnetlib.data.mdstore.modular.MDStoreActions"

		p:incomplete="false">

		<property name="actionMap">

			<map>

				<entry key="CREATE">

					<bean class="eu.dnetlib.data.mdstore.modular.CreateAction"

						p:profileCreator-ref="mdstoreProfileCreator"

						p:dao-ref="${services.mdstore.dao}" />

				</entry>

				<entry key="DELETE">

					<bean class="eu.dnetlib.data.mdstore.modular.DeleteAction"

						p:registryLocator-ref="registryLocator" p:dao-ref="${services.mdstore.dao}" />

				</entry>

				<entry key="FEED">

					<bean class="eu.dnetlib.data.mdstore.modular.FeedAction"

						p:dao-ref="${services.mdstore.dao}" 

						p:feeder-ref="mdstoreFeeder"/>

				</entry>

			</map>

		</property>

	</bean>

	<bean id="mdstoreProfileCreator" class="eu.dnetlib.data.mdstore.modular.MDStoreProfileCreator"

		p:registryLocator-ref="registryLocator" p:mdstoreDsTemplate-ref="mdstoreDsTemplate"

		p:endpoint-ref="mdStoreServiceEndpoint" p:eprBuilder-ref="jaxwsEndpointReferenceBuilder" />

	<bean id="mdstoreDsTemplate"

		class="eu.dnetlib.springutils.stringtemplate.StringTemplateFactory"

		p:template="classpath:/eu/dnetlib/data/mdstore/modular/mdstoreds-template.xml"

		scope="prototype" />

	<bean id="mdstoreFeeder" class="eu.dnetlib.data.mdstore.modular.MDStoreFeeder"

		p:dao-ref="${services.mdstore.dao}" p:resultSetClientFactory-ref="mdstoreResultSetClientFactory"

		p:syncFeed="${services.mdstore.syncFeed}" p:registryLocator-ref="registryLocator" 

		p:transactionManager-ref="${services.mdstore.transaction}"

		/>

	<bean id="mdstoreResultSetClientFactory" parent="resultSetClientFactory"

		p:pageSize="${services.mdstore.rsfactory.pagesize}" />

	<bean id="mdstoreRetriever" class="eu.dnetlib.data.mdstore.modular.MDStoreRetriever"

		p:dao-ref="${services.mdstore.dao}" p:resultSetFactory-ref="${services.mdstore.rsfactory}" />

	<!-- <bean id="mdstoreRecordParser" class="eu.dnetlib.data.mdstore.modular.SimpleRecordParser" 

		/> -->

	<bean id="mdstoreRecordParser" 

		factory-bean="recordParserFactory" factory-method="newInstance"/>

	<bean id="recordParserFactory" class="eu.dnetlib.data.mdstore.modular.RecordParserFactory"

		p:parserType="${services.mdstore.recordParser}" />

	<bean id="bulkRecordMapperFactory" class="eu.dnetlib.data.mdstore.modular.BulkRecordMapperFactory" />

	<!-- endpoints -->

	<jaxws:endpoint id="mdStoreServiceEndpoint" implementor="#mdStoreService"

		implementorClass="eu.dnetlib.data.mdstore.MDStoreService" address="/mdStore" />

	<template:instance name="serviceRegistrationManager"

		t:serviceRegistrationManagerClass="eu.dnetlib.enabling.tools.registration.ValidatingServiceRegistrationManagerImpl"

		t:name="mdStoreServiceRegistrationManager" t:service="mdStoreService"

		t:endpoint="mdStoreServiceEndpoint" t:jobScheduler="jobScheduler"

		t:serviceRegistrator="blackboardServiceRegistrator" />

</beans>

package eu.dnetlib.enabling.tools.blackboard;

import eu.dnetlib.data.mdstore.modular.MDStoreActions;

public class SampleCreateAction implements BlackboardServerAction<MDStoreActions> {

	@Override

	public void execute(BlackboardServerHandler handler, BlackboardJob job) {

		handler.done(job);

	}

}

package eu.dnetlib.data.mdstore.modular;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;

import eu.dnetlib.enabling.is.registry.ISRegistryDocumentNotFoundException;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;

import eu.dnetlib.enabling.tools.ServiceLocator;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.enabling.tools.blackboard.BlackboardServerAction;

import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;

public class DeleteAction extends AbstractMDStoreAction implements BlackboardServerAction<MDStoreActions> {

	private ServiceLocator<ISRegistryService> registryLocator;

	@Autowired

	private MDStoreTransactionManager transactionManager;

	@Override

	public void execute(final BlackboardServerHandler handler, final BlackboardJob job) throws ISRegistryDocumentNotFoundException, ISRegistryException {

		registryLocator.getService().deleteProfile(job.getParameters().get("id"));

		String currentId = job.getParameters().get("id");

		try {

			transactionManager.dropMDStore(currentId);

		} catch (MDStoreServiceException e) {

			throw new ISRegistryException("Error deleting mdstore with id " + currentId, e);

		}

		getDao().deleteMDStore(currentId);

		handler.done(job);

	}

	public ServiceLocator<ISRegistryService> getRegistryLocator() {

		return registryLocator;

	}

	@Required

	public void setRegistryLocator(final ServiceLocator<ISRegistryService> registryLocator) {

		this.registryLocator = registryLocator;

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:p="http://www.springframework.org/schema/p"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">

	<bean id="blackboardHandler" class="eu.dnetlib.test.utils.MockBeanFactory"

		p:clazz="eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler" />

	<bean id="executor"

		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerActionExecutor"

		p:blackboardHandler-ref="blackboardHandler" p:actionType="eu.dnetlib.data.mdstore.modular.MDStoreActions"

		p:incomplete="true">

		<property name="actionMap">

			<map>

				<entry key="CREATE">

					<bean class="eu.dnetlib.enabling.tools.blackboard.SampleCreateAction" />

				</entry>

			</map>

		</property>

	</bean>

</beans>

package eu.dnetlib.data.mdstore.modular;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.enabling.tools.blackboard.BlackboardServerAction;

import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;

public class FeedAction extends AbstractMDStoreAction implements BlackboardServerAction<MDStoreActions> {

	private static final Log log = LogFactory.getLog(FeedAction.class);

	private MDStoreFeeder feeder;

	@Override

	public void execute(final BlackboardServerHandler handler, final BlackboardJob job) throws Exception {

		final String mdId = job.getParameters().get("mdId");

		if ((mdId == null) || mdId.isEmpty()) throw new MDStoreServiceException("Blackboard param (mdId) is empty");

		final String epr = job.getParameters().get("epr");

		if ((epr == null) || epr.isEmpty()) throw new MDStoreServiceException("Blackboard param (mdId) is empty");

		String storingType = job.getParameters().get("storingType");

		if ((storingType == null) || storingType.isEmpty()) {

			storingType = "REFRESH";

		}

		feeder.feed(mdId, epr, storingType, true, new FeedDoneCallback() {

			@Override

			public void call(final int size) {

				job.getParameters().put("total", "" + size);

				handler.done(job);

			}

		}, new FeedFailedCallback() {

			@Override

			public void call(final Throwable e) {

				log.error("Error feeding mdstore: " + mdId, e);

				handler.failed(job, e);

			}

		});

	}

	public MDStoreFeeder getFeeder() {

		return feeder;

	}

	@Required

	public void setFeeder(final MDStoreFeeder feeder) {

		this.feeder = feeder;

	}

}

package eu.dnetlib.data.mdstore.modular;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.enabling.tools.blackboard.BlackboardServerAction;

import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;

public class CreateAction extends AbstractMDStoreAction implements BlackboardServerAction<MDStoreActions> {

	private MDStoreProfileCreator profileCreator;

	@Autowired

	private MDStoreTransactionManager transactionManager;

	@Override

	public void execute(final BlackboardServerHandler handler, final BlackboardJob job) throws ISRegistryException {

		final String format = job.getParameters().get("format");

		final String interpretation = job.getParameters().get("interpretation");

		final String layout = job.getParameters().get("layout");

		final String mdId = profileCreator.registerProfile(format, interpretation, layout);

		try {

			transactionManager.createMDStore(mdId);

		} catch (MDStoreServiceException e) {

			throw new ISRegistryException("Error on create mdStore", e);

		}

		getDao().createMDStore(mdId, format, interpretation, layout);

		job.getParameters().put("id", mdId);

		handler.done(job);

	}

	public MDStoreProfileCreator getProfileCreator() {

		return profileCreator;

	}

	@Required

	public void setProfileCreator(final MDStoreProfileCreator profileCreator) {

		this.profileCreator = profileCreator;

	}

}

package eu.dnetlib.data.mdstore.modular;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import eu.dnetlib.data.mdstore.DocumentNotFoundException;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.connector.MDStore;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;

import eu.dnetlib.enabling.resultset.ResultSetFactory;

public class MDStoreRetriever {

	private MDStoreDao dao;

	private ResultSetFactory resultSetFactory;

	@Autowired

	private MDStoreTransactionManager transactionManager;

	public W3CEndpointReference deliver(final String mdId, final String from, final String until, final String recordFilter) {

		String currentId;

		try {

			currentId = transactionManager.readMdStore(mdId);

			final MDStore mdStore = dao.getMDStore(currentId);

			return getResultSetFactory().createResultSet(mdStore.deliver(from, until, recordFilter));

		} catch (MDStoreServiceException e) {

			return null;

		}

	}

	public Iterable<String> deliver(final String format, final String layout, final String interpretation) {

		return Iterables.concat(Iterables.transform(dao.listMDStores(format, layout, interpretation), new Function<String, Iterable<String>>() {

			@Override

			public Iterable<String> apply(final String mdId) {

				try {

					String currentId = transactionManager.readMdStore(mdId);

					return dao.getMDStore(currentId).iterate();

				} catch (MDStoreServiceException e) {

					return null;

				}

			}

		}));

	}

	public String deliverRecord(final String mdId, final String recordId) throws DocumentNotFoundException {

		try {

			String currentId = transactionManager.readMdStore(mdId);

			return dao.getMDStore(currentId).getRecord(recordId);

		} catch (MDStoreServiceException e) {

			return null;

		}

	}

	public MDStoreDao getDao() {

		return dao;

	}

	@Required

	public void setDao(final MDStoreDao dao) {

		this.dao = dao;

	}

	@Required

	public void setResultSetFactory(final ResultSetFactory resultSetFactory) {

		this.resultSetFactory = resultSetFactory;

	}

	public ResultSetFactory getResultSetFactory() {

		return resultSetFactory;

	}

}

package eu.dnetlib.data.mdstore.modular.connector;

import java.util.List;

public interface MDStoreDao {

	MDStore getMDStore(String mdId);

	List<String> listMDStores();

	List<String> listMDStores(String format, String layout, String interpretation);

	void createMDStore(String mdId, String format, String interpretation, String layout);

	void deleteMDStore(String id);

}

package eu.dnetlib.data.mdstore.modular.connector;

import java.util.Date;

public class MDStoreTransactionInfo {

	private String id;

	private Boolean refresh;

	private Date date;

	private long size;

	/**

	 * @return the date

	 */

	public Date getDate() {

		return date;

	}

	/**

	 * @param date

	 *            the date to set

	 */

	public void setDate(final Date date) {

		this.date = date;

	}

	/**

	 * @return the id

	 */

	public String getId() {

		return id;

	}

	/**

	 * @param id

	 *            the id to set

	 */

	public void setId(final String id) {

		this.id = id;

	}

	/**

	 * @return the refresh

	 */

	public Boolean getRefresh() {

		return refresh;

	}

	/**

	 * @param refresh

	 *            the refresh to set

	 */

	public void setRefresh(final Boolean refresh) {

		this.refresh = refresh;

	}

	/**

	 * @return the size

	 */

	public long getSize() {

		return size;

	}

	/**

	 * @param size

	 *            the size to set

	 */

	public void setSize(final long size) {

		this.size = size;

	}

}

package eu.dnetlib.data.mdstore.modular.connector;

import java.util.ArrayList;

import java.util.List;

public class MDStoreManagerInfo {

	private String mdId;

	private String currentId;

	private List<MDStoreExpiredInfo> stillUsed;

	private List<MDStoreTransactionInfo> transactions;

	public String getMdId() {

		return mdId;

	}

	public void setMdId(final String mdId) {

		this.mdId = mdId;

	}

	public String getCurrentId() {

		return currentId;

	}

	public void setCurrentId(final String currentId) {

		this.currentId = currentId;

	}

	public List<MDStoreExpiredInfo> getStillUsed() {

		return stillUsed;

	}

	public void setStillUsed(final List<MDStoreExpiredInfo> stillUsed) {

		this.stillUsed = stillUsed;

	}

	public List<MDStoreTransactionInfo> getTransactions() {

		return transactions;

	}

	public void setTransactions(final List<MDStoreTransactionInfo> transactions) {

		this.transactions = transactions;

	}

	public void addExpiredItem(final MDStoreExpiredInfo info) {

		if (stillUsed == null) {

			stillUsed = new ArrayList<MDStoreExpiredInfo>();

		}

		stillUsed.add(info);

	}

	public void addTransactionInfo(final MDStoreTransactionInfo info) {

		if (transactions == null) {

			transactions = new ArrayList<MDStoreTransactionInfo>();

		}

		transactions.add(info);

	}

}

package eu.dnetlib.data.mdstore.modular.connector;

import java.util.Date;

public class MDStoreExpiredInfo {

	private String id;

	private Date lastRead;

	/**

	 * @return the lastRead

	 */

	public Date getLastRead() {

		return lastRead;

	}

	/**

	 * @param lastRead

	 *            the lastRead to set

	 */

	public void setLastRead(final Date lastRead) {

		this.lastRead = lastRead;

	}

	/**

	 * @return the id

	 */

	public String getId() {

		return id;

	}

	/**

	 * @param id

	 *            the id to set

	 */

	public void setId(final String id) {

		this.id = id;

	}

}

package eu.dnetlib.data.mdstore.modular.connector;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

public interface MDStoreTransactionManager {

	/**

	 * Creates the md store.

	 * 

	 * @param mdId

	 *            the md id

	 * @throws MDStoreServiceException

	 *             the MD store service exception

	 */

	public abstract void createMDStore(String mdId) throws MDStoreServiceException;

	/**

	 * Drop md store.

	 * 

	 * @param mdId

	 *            the md id

	 * @throws MDStoreServiceException

	 *             the MD store service exception

	 */

	public abstract void dropMDStore(String mdId) throws MDStoreServiceException;

	/**

	 * Gets the MD store collection.

	 * 

	 * @param mdId

	 *            the md id

	 * @return the MD store collection

	 * @throws MDStoreServiceException

	 *             the MD store service exception

	 */

	public abstract String getMDStoreCollection(String mdId) throws MDStoreServiceException;

	/**

	 * Start a new transaction for writing in the mdstore this will create a temporary mdstore in which save the content. and after finished

	 * switch it to the existing one

	 * 

	 * @param mdId

	 * @param refresh

	 * @throws MDStoreServiceException

	 *             if the mdStore Id doesn't exists in the metadata

	 */

	public abstract String startTransaction(String mdId, boolean refresh) throws MDStoreServiceException;

	/**

	 * Commit the transaction

	 * 

	 * 

	 * @param transactionId

	 * @param mdId

	 * @return

	 * @throws MDStoreServiceException

	 */

	public abstract boolean commit(String transactionId, String mdId) throws MDStoreServiceException;

	/**

	 * Book a a current mdstore, so the transaction manager can't delete during a possible commit, until the resultset associate has not

	 * expired

	 * 

	 * @param mdStoreId

	 * @return

	 * @throws MDStoreServiceException

	 */

	public abstract String readMdStore(String mdStoreId) throws MDStoreServiceException;

	/**

	 * Return a JSON about the transaction assigned to a particular mdstore

	 * 

	 * @param mdStoreId

	 * @return

	 * @throws MDStoreServiceException

	 */

	public abstract MDStoreManagerInfo getInfoForCurrentMdStore(String mdStoreId) throws MDStoreServiceException;

	/**

	 * Manually drop an old collection assigned to a particular mdStore

	 * 

	 * @param mdId

	 *            : the id of the mdStore

	 * @param idToDrop

	 *            : The id of the collection to be dropped

	 * @return

	 * @throws MDStoreServiceException

	 */

	public abstract Boolean dropUsed(String mdId, String idToDrop) throws MDStoreServiceException;

	/**

	 * Manually drop an old collection assigned to a particular mdStore

	 * 

	 * @param mdId

	 *            : the id of the mdStore

	 * @param idToDrop

	 *            : The id of the collection to be dropped

	 * @return

	 * @throws MDStoreServiceException

	 */

	public abstract Boolean dropTransaction(String mdId, String idToDrop) throws MDStoreServiceException;

	/**

	 * Start the garbage collection of the old mdstore not used

	 * 

	 * @throws MDStoreServiceException

	 */

	public abstract void garbage() throws MDStoreServiceException;

}

package eu.dnetlib.data.mdstore.modular.connector;

import eu.dnetlib.data.mdstore.DocumentNotFoundException;

import eu.dnetlib.enabling.resultset.ResultSetListener;

public interface MDStore {

	String getId();

	String getFormat();

	String getInterpretation();

	String getLayout();

	void truncate();

	int feed(Iterable<String> records, boolean incremental);

	ResultSetListener deliver(String from, String until, String recordFilter);

	ResultSetListener deliverIds(String from, String until, String recordFilter);

	Iterable<String> iterate();

	int getSize();

	void deleteRecord(String recordId);

	String getRecord(String recordId) throws DocumentNotFoundException;

}

package eu.dnetlib.data.mdstore.modular.connector;

public class MDStoreResults {

	private Iterable<String> records;

	private int size;

	public MDStoreResults(Iterable<String> records, int size) {

		super();

		this.records = records;

		this.size = size;

	}

	public int getSize() {

		return size;

	}

	public void setSize(int size) {

		this.size = size;

	}

	public Iterable<String> getRecords() {

		return records;

	}

	public void setRecords(Iterable<String> records) {

		this.records = records;

	}

}

package eu.dnetlib.data.mdstore.modular;

import java.util.List;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.data.mdstore.DocumentNotFoundException;

import eu.dnetlib.data.mdstore.MDStoreService;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.enabling.resultset.IterableResultSetFactory;

import eu.dnetlib.enabling.tools.AbstractBaseService;

import eu.dnetlib.enabling.tools.blackboard.NotificationHandler;

public class ModularMDStoreService extends AbstractBaseService implements MDStoreService {

	/**

	 * notification handler.

	 */

	private NotificationHandler notificationHandler;

	private MDStoreFeeder feeder;

	private MDStoreRetriever retriever;

	private IterableResultSetFactory iterableResultSetFactory;

	@Override

	public W3CEndpointReference deliverMDRecords(final String mdId, final String from, final String until, final String recordFilter)

			throws MDStoreServiceException {

		return retriever.deliver(mdId, from, until, recordFilter);

	}

	@Override

	public W3CEndpointReference bulkDeliverMDRecords(final String format, final String layout, final String interpretation) throws MDStoreServiceException {

		return getIterableResultSetFactory().createIterableResultSet(retriever.deliver(format, layout, interpretation));

	}

	@Override

	public String deliverRecord(final String mdId, final String recordId) throws DocumentNotFoundException {

		return retriever.deliverRecord(mdId, recordId);

	}

	@Override

	public List<String> getListOfMDStores() throws MDStoreServiceException {

		return retriever.getDao().listMDStores();

	}

	@Override

	public List<String> listMDStores(final String format, final String layout, final String interpretation) throws MDStoreServiceException {

		return retriever.getDao().listMDStores(format, layout, interpretation);

	}

	@Override

	public void notify(final String subscriptionId, final String topic, final String isId, final String message) {

		getNotificationHandler().notified(subscriptionId, topic, isId, message);

	}

	@Override

	public boolean storeMDRecordsFromRS(final String mdId, final String rsId, final String storingType) throws MDStoreServiceException {

		throw new MDStoreServiceException("not implemented, use the Blackboard asynchronous equivalent");

	}

	public NotificationHandler getNotificationHandler() {

		return notificationHandler;

	}

	@Required

	public void setNotificationHandler(final NotificationHandler notificationHandler) {

		this.notificationHandler = notificationHandler;

	}

	public MDStoreFeeder getFeeder() {

		return feeder;

	}

	public void setFeeder(final MDStoreFeeder feeder) {

		this.feeder = feeder;

	}

	public MDStoreRetriever getRetriever() {

		return retriever;

	}

	public void setRetriever(final MDStoreRetriever retriever) {

		this.retriever = retriever;

	}

	public IterableResultSetFactory getIterableResultSetFactory() {

		return iterableResultSetFactory;

	}

	@Required

	public void setIterableResultSetFactory(final IterableResultSetFactory iterableResultSetFactory) {

		this.iterableResultSetFactory = iterableResultSetFactory;

	}

}

package eu.dnetlib.data.mdstore.modular;

public interface FeedFailedCallback {

	void call(Throwable e);

}

package eu.dnetlib.data.mdstore.modular;

public interface FeedDoneCallback {

	void call(int size);

}

package eu.dnetlib.data.mdstore.modular;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.connector.MDStore;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;

import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;

import eu.dnetlib.enabling.tools.ServiceLocator;

import eu.dnetlib.miscutils.datetime.DateUtils;

public class MDStoreFeeder {

	private static final Log log = LogFactory.getLog(MDStoreFeeder.class);

	private MDStoreDao dao;

	private ResultSetClientFactory resultSetClientFactory;

	private ServiceLocator<ISRegistryService> registryLocator;

	private MDStoreTransactionManager transactionManager;

	private boolean syncFeed = true;

	public void feed(final String mdId,

			final String rsEpr,

			final String storingType,

			final boolean sync,

			final FeedDoneCallback doneCallback,

			final FeedFailedCallback failCallback) throws MDStoreServiceException {

		log.info("Start feeding mdstore " + mdId + " with epr " + rsEpr);

		String temporaryID = "";

		try {

			final boolean incremental = !"REFRESH".equals(storingType);

			temporaryID = transactionManager.startTransaction(mdId, !incremental);

			final MDStore mdstore = dao.getMDStore(temporaryID);

			final Iterable<String> records = resultSetClientFactory.getClient(rsEpr);

			if (!incremental) {

				mdstore.truncate();

			}

			final int size = mdstore.feed(records, incremental);

			transactionManager.commit(temporaryID, mdId);

			touch(mdId, size);

			log.info("Finished feeding mdstore " + mdId + " - new size: " + size);

			doneCallback.call(size);

		} catch (Exception e) {

			log.error("Error feeding mdstore: " + mdId + ", temporary id: " + temporaryID);

			failCallback.call(e);

		}

	}

	/**

	 * Sets the last modified date in the profile.

	 * 

	 * @param mdId

	 */

	public void touch(final String mdId, final int size) {

		try {

			final String now = DateUtils.now_ISO8601();

			final String mdstoreXUpdate =

					"for $x in //RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + mdId + "']" + "return update value $x//LAST_STORAGE_DATE with '" + now

							+ "'";

			registryLocator.getService().executeXUpdate(mdstoreXUpdate);

			touchSize(mdId, size);

		} catch (final Exception e) {

			throw new IllegalStateException(e);

		}

	}

	public void touchSize(final String mdId, final int size) {

		try {

			final String mdstoreNumberXUpdate =

					"for $x in //RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + mdId + "']" + "return update value $x//NUMBER_OF_RECORDS with '" + size

							+ "'";

			registryLocator.getService().executeXUpdate(mdstoreNumberXUpdate);

		} catch (final Exception e) {

			throw new IllegalStateException(e);

		}

	}

	public MDStoreDao getDao() {

		return dao;

	}

	@Required

	public void setDao(final MDStoreDao dao) {

		this.dao = dao;

	}

	public ResultSetClientFactory getResultSetClientFactory() {

		return resultSetClientFactory;

	}

	@Required

	public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {

		this.resultSetClientFactory = resultSetClientFactory;

	}

	public ServiceLocator<ISRegistryService> getRegistryLocator() {

		return registryLocator;

	}

	@Required

	public void setRegistryLocator(final ServiceLocator<ISRegistryService> registryLocator) {

		this.registryLocator = registryLocator;

	}

	public boolean isSyncFeed() {

		return syncFeed;

	}

	public void setSyncFeed(final boolean syncFeed) {

		this.syncFeed = syncFeed;

	}

	/**

	 * @return the transactionManager

	 */

	public MDStoreTransactionManager getTransactionManager() {

		return transactionManager;

	}

	/**

	 * @param transactionManager

	 *            the transactionManager to set

	 */

	@Required

	public void setTransactionManager(final MDStoreTransactionManager transactionManager) {

		this.transactionManager = transactionManager;

	}

}

package eu.dnetlib.data.mdstore.modular;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

public abstract class AbstractMDStoreAction {

	private MDStoreDao dao;

	public MDStoreDao getDao() {

		return dao;

	}

	public void setDao(MDStoreDao dao) {

		this.dao = dao;

	}

}

package eu.dnetlib.data.mdstore.modular;

import java.io.ByteArrayInputStream;

import java.util.HashMap;

import java.util.Map;

import java.util.Stack;

import javax.xml.stream.XMLInputFactory;

import javax.xml.stream.XMLStreamConstants;

import javax.xml.stream.XMLStreamException;

import javax.xml.stream.XMLStreamReader;

/**

 * This method outperforms SimpleRecordParser by a vast amount, especially since we are just getting stuff in the

 * header.

 * 

 * @author marko

 * 

 */

public class StreamingRecordParser implements RecordParser {

	@Override

	public Map<String, String> parseRecord(String record) {

		try {

			XMLInputFactory factory = XMLInputFactory.newInstance();

			XMLStreamReader parser = factory.createXMLStreamReader(new ByteArrayInputStream(record.getBytes()));

			HashMap<String, String> res = new HashMap<String, String>();

			Stack<String> elementStack = new Stack<String>();

			elementStack.push("/");

			while (parser.hasNext()) {

				int event = parser.next();

				if (event == XMLStreamConstants.END_ELEMENT) {

					elementStack.pop();

				} else if (event == XMLStreamConstants.START_ELEMENT) {

					final String localName = parser.getLocalName();

					elementStack.push(localName);

					if ("objIdentifier".equals(localName)) {

						parser.next();

						res.put("id", parser.getText().trim());

					} else if ("identifier".equals(localName) && "efgEntity".equals(grandParent(elementStack))) {

						if (!res.containsKey("originalId")) {

							parser.next();

//							log.info("ZZZZZZ OK: found identifier at right depth " + elementStack);

							res.put("originalId", parser.getText().trim());

						}

					}

					else if ("identifier".equals(localName)) {

//						log.info("ZZZZZZ: found identifier not at right depth " + elementStack + " grand parent " + grandParent(elementStack));

					}

					if (res.containsKey("id") && res.containsKey("originalId"))

						return res;

				}

			}

			return res;

		} catch (XMLStreamException e) {

			throw new IllegalStateException(e);

		}

	}

	private String grandParent(Stack<String> elementStack) {

		if (elementStack.size() <= 3)

			return "";

		return elementStack.get(elementStack.size() - 3);

	}

}

package eu.dnetlib.data.mdstore.modular;

import java.io.StringReader;

import java.util.HashMap;

import java.util.Map;

import javax.xml.xpath.XPath;

import javax.xml.xpath.XPathFactory;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.xml.sax.InputSource;

/**

 * Terrible implementation of a record parser.

 * 

 * @author marko

 *

 */

public class SimpleRecordParser implements RecordParser {

	static final Log log = LogFactory.getLog(SimpleRecordParser.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Override

	public Map<String, String> parseRecord(String record) {

		Map<String, String> props = new HashMap<String, String>();

		try {

//			DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();

			XPath xpath = XPathFactory.newInstance().newXPath();

//			Document doc = builder.parse(new InputSource(new StringReader(record)));

			InputSource doc = new InputSource(new StringReader(record));

			props.put("id", xpath.evaluate("//*[local-name()='objIdentifier']", doc));

			props.put("originalId", xpath.evaluate("//*[local-name()='efgEntity']/*/*[local-name()='identifier']", doc));

//			String date = xpath.evaluate("//*[local-name()='dateOfCollection'][1]", doc);

//			props.put("date", new Date(date).getTime());	

		} catch (Exception e) {

			log.warn("got exception while parsing document", e);

			log.warn("record is:");

			log.warn(record);

			log.warn("------------");

		}

		return props;

	}

}

package eu.dnetlib.data.mdstore.modular;

import javax.xml.ws.Endpoint;

import org.antlr.stringtemplate.StringTemplate;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;

import eu.dnetlib.enabling.tools.ServiceLocator;

import eu.dnetlib.soap.EndpointReferenceBuilder;

public class MDStoreProfileCreator {

	/**

	 * registry service locator.

	 */

	private ServiceLocator<ISRegistryService> registryLocator;

	/**

	 * mdstore ds template.

	 */

	private StringTemplate mdstoreDsTemplate;

	/**

	 * service endpoint.

	 */

	private Endpoint endpoint;

	/**

	 * endpoint builder.

	 */

	private EndpointReferenceBuilder<Endpoint> eprBuilder;

	public String registerProfile(String format, String interpretation, String layout) throws ISRegistryException {

		// XXX: mini hack

		StringTemplate template = new StringTemplate(mdstoreDsTemplate.getTemplate());

		template.setAttribute("serviceUri", eprBuilder.getAddress(endpoint));

		template.setAttribute("format", format);

		template.setAttribute("interpretation", interpretation);

		template.setAttribute("layout", layout);

		return registryLocator.getService().registerProfile(template.toString());

	}

	public ServiceLocator<ISRegistryService> getRegistryLocator() {

		return registryLocator;

	}

	@Required

	public void setRegistryLocator(ServiceLocator<ISRegistryService> registryLocator) {

		this.registryLocator = registryLocator;

	}

	public StringTemplate getMdstoreDsTemplate() {

		return mdstoreDsTemplate;

	}

	@Required

	public void setMdstoreDsTemplate(StringTemplate mdstoreDsTemplate) {

		this.mdstoreDsTemplate = mdstoreDsTemplate;

	}

	public Endpoint getEndpoint() {

		return endpoint;

	}

	@Required

	public void setEndpoint(Endpoint endpoint) {

		this.endpoint = endpoint;

	}

	public EndpointReferenceBuilder<Endpoint> getEprBuilder() {

		return eprBuilder;

	}

	@Required

	public void setEprBuilder(EndpointReferenceBuilder<Endpoint> eprBuilder) {

		this.eprBuilder = eprBuilder;

	}

}

package eu.dnetlib.data.mdstore.modular;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.miscutils.factory.Factory;

public class RecordParserFactory implements Factory<RecordParser> {

	private Class<? extends RecordParser> parserType;

	@Override

	public RecordParser newInstance() {

		try {

			return getParserType().newInstance();

		} catch (Throwable e) {

			throw new RuntimeException(e);

		} 	

	}

	@Required

	public void setParserType(Class<? extends RecordParser> parserType) {

		this.parserType = parserType;

	}

	public Class<? extends RecordParser> getParserType() {

		return parserType;

	}

}

package eu.dnetlib.data.mdstore.modular;

import java.io.StringReader;

import java.io.StringWriter;

import java.util.List;

import javax.xml.namespace.QName;

import javax.xml.stream.XMLEventFactory;

import javax.xml.stream.XMLEventReader;

import javax.xml.stream.XMLEventWriter;

import javax.xml.stream.XMLInputFactory;

import javax.xml.stream.XMLOutputFactory;

import javax.xml.stream.XMLStreamException;

import javax.xml.stream.events.Namespace;

import javax.xml.stream.events.StartElement;

import javax.xml.stream.events.XMLEvent;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.google.common.base.Function;

import com.google.common.collect.Lists;

import eu.dnetlib.miscutils.collections.Pair;

import eu.dnetlib.miscutils.factory.Factory;

public class BulkRecordMapperFactory implements Factory<Function<String, Pair<String, String>>> {

	private static final Log log = LogFactory.getLog(BulkRecordMapperFactory.class); // NOPMD by marko on 11/24/08 5:02 PM

	protected static final String MD_RECORD = "mdRecord";

	protected static final String MD_ID = "mdId";

	protected static final String RECORD = "record";

	protected ThreadLocal<XMLInputFactory> inputFactory = new ThreadLocal<XMLInputFactory>() {

		@Override

		protected XMLInputFactory initialValue() {

			return XMLInputFactory.newInstance();

		}

	};

	protected ThreadLocal<XMLOutputFactory> outputFactory = new ThreadLocal<XMLOutputFactory>() {

		@Override

		protected XMLOutputFactory initialValue() {

			return XMLOutputFactory.newInstance();

		}

	};

	protected ThreadLocal<XMLEventFactory> eventFactory = new ThreadLocal<XMLEventFactory>() {

		@Override

		protected XMLEventFactory initialValue() {

			return XMLEventFactory.newInstance();

		}

	};

	@Override

	public Function<String, Pair<String, String>> newInstance() {

		return new Function<String, Pair<String, String>>() {

			private String mdId = null;

			private String record = null;

			@Override

			public Pair<String, String> apply(String embeddedRecord) {

				try {

					final XMLEventReader parser = inputFactory.get().createXMLEventReader(new StringReader(embeddedRecord));

					while (parser.hasNext()) {