D-Net

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>cnr-modular-mdstore-service</artifactId>

	<packaging>jar</packaging>

	<version>4.0.4-SNAPSHOT</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/cnr-modular-mdstore-service/trunk</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.springframework</groupId>

			<artifactId>spring-test</artifactId>

			<version>${spring.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.mockito</groupId>

			<artifactId>mockito-core</artifactId>

			<version>1.6</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-test-utils</artifactId>

			<version>[1.0.0,2.0.0)</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-rmi-api</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-blackboard-common</artifactId>

			<version>[2.2.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-misc-utils</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-resultset-service</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-resultset-client</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

	</dependencies>

</project>

/**

 * 

 */

package eu.dnetlib.data.mdstore.modular;

public enum MDStoreActions {

	CREATE,

	DELETE,

	FEED

}

package eu.dnetlib.data.mdstore.modular.connector;

import java.util.List;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.MDStoreDescription;

public interface MDStoreDao {

	MDStore getMDStore(String mdId) throws MDStoreServiceException;

	MDStore readMDStore(String mdId) throws MDStoreServiceException;

	MDStore startTransaction(String mdId, boolean refresh) throws MDStoreServiceException;

	void commit(String transactionId, String mdId) throws MDStoreServiceException;

	List<MDStoreDescription> listMDStores() throws MDStoreServiceException;

	List<String> listMDStores(String format, String layout, String interpretation) throws MDStoreServiceException;

	void createMDStore(String mdId, String format, String interpretation, String layout) throws MDStoreServiceException;

	void deleteMDStore(String id) throws MDStoreServiceException;

	int getCachedSize(String id) throws MDStoreServiceException;

	void refreshSizes() throws MDStoreServiceException;

	int refreshSize(String id) throws MDStoreServiceException;

	void startGarbage() throws MDStoreServiceException;

	MDStoreDBStatus getDBStatus();

}

package eu.dnetlib.data.mdstore.modular.connector;

public class MDStoreDBStatus {

	private int handledDatastructures;

	private int usedDiskSpace;

	private String date;

	public MDStoreDBStatus() {}

	public MDStoreDBStatus(final int handledDatastructures, final int usedDiskSpace, final String date) {

		this.handledDatastructures = handledDatastructures;

		this.usedDiskSpace = usedDiskSpace;

		this.date = date;

	}

	public int getHandledDatastructures() {

		return handledDatastructures;

	}

	public void setHandledDatastructures(final int handledDatastructures) {

		this.handledDatastructures = handledDatastructures;

	}

	public int getUsedDiskSpace() {

		return usedDiskSpace;

	}

	public void setUsedDiskSpace(final int usedDiskSpace) {

		this.usedDiskSpace = usedDiskSpace;

	}

	public String getDate() {

		return date;

	}

	public void setDate(final String date) {

		this.date = date;

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws"

	xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing"

	xmlns:p="http://www.springframework.org/schema/p" xmlns:http="http://cxf.apache.org/transports/http/configuration"

	xmlns:t="http://dnetlib.eu/springbeans/t" xmlns:template="http://dnetlib.eu/springbeans/template"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd

                                    http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd

                                    http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd

                                    http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd

                            http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd

                            http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">

	<!-- beans -->

	<bean id="mdStoreService" class="eu.dnetlib.data.mdstore.modular.ModularMDStoreService"

		init-method="start" destroy-method="stop" p:notificationHandler-ref="mdstoreNotificationHandler"

		p:iterableResultSetFactory-ref="iterableResultSetFactory"

		p:feeder-ref="mdstoreFeeder" p:retriever-ref="mdstoreRetriever" />

	<bean id="mdstoreNotificationHandler"

		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerExecutorNotificationHandler"

		p:blackboardExecutor-ref="mdstoreBlackboardExecutor" />

	<bean id="mdstoreBlackboardExecutor"

		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerActionExecutor"

		p:blackboardHandler-ref="blackboardHandler"

		p:actionType="eu.dnetlib.data.mdstore.modular.MDStoreActions"

		p:incomplete="false">

		<property name="actionMap">

			<map>

				<entry key="CREATE">

					<bean class="eu.dnetlib.data.mdstore.modular.CreateAction"

						p:profileCreator-ref="mdstoreProfileCreator"

						p:dao-ref="${services.mdstore.dao}" />

				</entry>

				<entry key="DELETE">

					<bean class="eu.dnetlib.data.mdstore.modular.DeleteAction"

						p:serviceLocator-ref="uniqueServiceLocator" p:dao-ref="${services.mdstore.dao}" />

				</entry>

				<entry key="FEED">

					<bean class="eu.dnetlib.data.mdstore.modular.FeedAction"

						p:dao-ref="${services.mdstore.dao}" 

						p:feeder-ref="mdstoreFeeder"/>

				</entry>

			</map>

		</property>

	</bean>

	<bean id="mdstoreProfileCreator" class="eu.dnetlib.data.mdstore.modular.MDStoreProfileCreator"

		p:serviceLocator-ref="uniqueServiceLocator" p:mdstoreDsTemplate-ref="mdstoreDsTemplate"

		p:endpoint-ref="mdStoreServiceEndpoint" p:eprBuilder-ref="jaxwsEndpointReferenceBuilder" />

	<bean id="mdstoreDsTemplate"

		class="eu.dnetlib.springutils.stringtemplate.StringTemplateFactory"

		p:template="classpath:/eu/dnetlib/data/mdstore/modular/mdstoreds-template.xml"

		scope="prototype" />

	<bean id="mdstoreFeeder" class="eu.dnetlib.data.mdstore.modular.MDStoreFeeder"

		p:dao-ref="${services.mdstore.dao}" p:resultSetClientFactory-ref="mdstoreResultSetClientFactory"

		p:syncFeed="${services.mdstore.syncFeed}" p:serviceLocator-ref="uniqueServiceLocator" />

	<bean id="mdstoreResultSetClientFactory" parent="resultSetClientFactory"

		p:pageSize="${services.mdstore.rsfactory.pagesize}" />

	<bean id="mdstoreRetriever" class="eu.dnetlib.data.mdstore.modular.MDStoreRetriever"

		p:dao-ref="${services.mdstore.dao}" p:resultSetFactory-ref="${services.mdstore.rsfactory}" />

	<!-- <bean id="mdstoreRecordParser" class="eu.dnetlib.data.mdstore.modular.SimpleRecordParser" 

		/> -->

	<bean id="mdstoreRecordParser" 

		factory-bean="recordParserFactory" factory-method="newInstance"/>

	<bean id="recordParserFactory" class="eu.dnetlib.data.mdstore.modular.RecordParserFactory"

		p:parserType="${services.mdstore.recordParser}" />

	<bean id="bulkRecordMapperFactory" class="eu.dnetlib.data.mdstore.modular.BulkRecordMapperFactory" />

	<!-- endpoints -->

	<jaxws:endpoint id="mdStoreServiceEndpoint" implementor="#mdStoreService"

		implementorClass="eu.dnetlib.data.mdstore.MDStoreService" address="/mdStore" />

	<template:instance name="serviceRegistrationManager"

		t:serviceRegistrationManagerClass="eu.dnetlib.enabling.tools.registration.ValidatingServiceRegistrationManagerImpl"

		t:name="mdStoreServiceRegistrationManager" t:service="mdStoreService"

		t:endpoint="mdStoreServiceEndpoint" t:jobScheduler="jobScheduler"

		t:serviceRegistrator="blackboardServiceRegistrator" />

</beans>

<?xml version="1.0"?>

<RESOURCE_PROFILE>

	<HEADER>

		<RESOURCE_IDENTIFIER value="" />

		<RESOURCE_TYPE value="MDStoreDSResourceType" />

		<RESOURCE_KIND value="MDStoreDSResources" />

		<RESOURCE_URI value="$serviceUri$?wsdl" />

		<DATE_OF_CREATION value="" />

	</HEADER>

	<BODY>

		<CONFIGURATION>

			<METADATA_FORMAT>$format$</METADATA_FORMAT>

			<METADATA_FORMAT_INTERPRETATION>$interpretation$

			</METADATA_FORMAT_INTERPRETATION>

			<METADATA_FORMAT_LAYOUT>$layout$</METADATA_FORMAT_LAYOUT>

		</CONFIGURATION>

		<STATUS>

			<PENULTIMATE_STORAGE_DATE></PENULTIMATE_STORAGE_DATE>

			<LAST_STORAGE_DATE></LAST_STORAGE_DATE>

			<NUMBER_OF_RECORDS>0</NUMBER_OF_RECORDS>

			<FETCHING_FREQUENCY />

			<STATISTICS_FIELDS />

		</STATUS>

		<SECURITY_PARAMETERS />

	</BODY>

</RESOURCE_PROFILE>

package eu.dnetlib.data.mdstore.modular.connector;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

public interface MDStoreTransactionManager {

	/**

	 * Verify consistency.

	 *

	 * @throws MDStoreServiceException

	 *             the MD store service exception

	 */

	public abstract void verifyConsistency() throws MDStoreServiceException;

	/**

	 * Creates the md store.

	 *

	 * @param mdId

	 *            the md id

	 * @throws MDStoreServiceException

	 *             the MD store service exception

	 */

	public abstract void createMDStore(String mdId) throws MDStoreServiceException;

	/**

	 * Drop md store.

	 *

	 * @param mdId

	 *            the md id

	 * @throws MDStoreServiceException

	 *             the MD store service exception

	 */

	public abstract void dropMDStore(String mdId) throws MDStoreServiceException;

	/**

	 * Gets the MD store collection.

	 *

	 * @param mdId

	 *            the md id

	 * @return the MD store collection

	 * @throws MDStoreServiceException

	 *             the MD store service exception

	 */

	public abstract String getMDStoreCollection(String mdId) throws MDStoreServiceException;

	/**

	 * Start a new transaction for writing in the mdstore this will create a temporary mdstore in which save the content. and after finished

	 * switch it to the existing one

	 *

	 * @param mdId

	 * @param refresh

	 * @throws MDStoreServiceException

	 *             if the mdStore Id doesn't exists in the metadata

	 */

	public abstract String startTransaction(String mdId, boolean refresh) throws MDStoreServiceException;

	/**

	 * Commit the transaction.

	 *

	 * @param transactionId

	 *            the transaction id

	 * @param mdstoreId

	 *            the mdstore id

	 * @param currentMDStore

	 *            the current md store

	 * @return true, if successful

	 * @throws MDStoreServiceException

	 *             the MD store service exception

	 */

	public abstract boolean commit(String transactionId, String mdstoreId, MDStore currentMDStore) throws MDStoreServiceException;

	/**

	 * Book a a current mdstore, so the transaction manager can't delete during a possible commit, until the resultset associate has not

	 * expired

	 *

	 * @param mdStoreId

	 * @return

	 * @throws MDStoreServiceException

	 */

	public abstract String readMdStore(String mdStoreId) throws MDStoreServiceException;

	/**

	 * Return a JSON about the transaction assigned to a particular mdstore

	 *

	 * @param mdStoreId

	 * @return

	 * @throws MDStoreServiceException

	 */

	public abstract MDStoreManagerInfo getInfoForCurrentMdStore(String mdStoreId) throws MDStoreServiceException;

	/**

	 * Manually drop an old collection assigned to a particular mdStore

	 *

	 * @param mdId

	 *            : the id of the mdStore

	 * @param idToDrop

	 *            : The id of the collection to be dropped

	 * @return

	 * @throws MDStoreServiceException

	 */

	public abstract Boolean dropUsed(String mdId, String idToDrop) throws MDStoreServiceException;

	/**

	 * Manually drop an old collection assigned to a particular mdStore

	 *

	 * @param mdId

	 *            : the id of the mdStore

	 * @param idToDrop

	 *            : The id of the collection to be dropped

	 * @return

	 * @throws MDStoreServiceException

	 */

	public abstract Boolean dropTransaction(String mdId, String idToDrop) throws MDStoreServiceException;

	/**

	 * Start the garbage collection of the old mdstore not used

	 *

	 * @throws MDStoreServiceException

	 */

	public abstract void garbage() throws MDStoreServiceException;

}

package eu.dnetlib.data.mdstore.modular.connector;

import eu.dnetlib.data.mdstore.DocumentNotFoundException;

import eu.dnetlib.enabling.resultset.ResultSetListener;

public interface MDStore {

	String getId();

	String getFormat();

	String getInterpretation();

	String getLayout();

	void truncate();

	void feed(Iterable<String> records, boolean incremental);

	ResultSetListener deliver(String from, String until, String recordFilter);

	ResultSetListener deliverIds(String from, String until, String recordFilter);

	Iterable<String> iterate();

	int getSize();

	void deleteRecord(String recordId);

	String getRecord(String recordId) throws DocumentNotFoundException;

}

package eu.dnetlib.enabling.tools.blackboard;

import static org.junit.Assert.assertNotNull;

import static org.mockito.Matchers.anyObject;

import static org.mockito.Matchers.eq;

import static org.mockito.Mockito.mock;

import static org.mockito.Mockito.verify;

import static org.mockito.Mockito.when;

import javax.annotation.Resource;

import org.junit.Before;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.springframework.test.context.ContextConfiguration;

import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import eu.dnetlib.data.mdstore.modular.MDStoreActions;

@RunWith(SpringJUnit4ClassRunner.class)

@ContextConfiguration

public class BlackboardServerActionExecutorTest {

	@Resource

	public transient BlackboardServerHandler blackboardHandler;

	@Resource

	public transient BlackboardServerActionExecutor<MDStoreActions> executor;

	@Before

	public void setUp() throws Exception {

	}

	@Test

	public void testExecutor() {

		assertNotNull(executor);

		BlackboardJob job = mock(BlackboardJob.class);

		when(job.getAction()).thenReturn("CREATE");

		executor.execute(job);

		verify(blackboardHandler).done(eq(job));

	}

	@Test

	public void testExecutorUnimplemented() {

		assertNotNull(executor);

		BlackboardJob job = mock(BlackboardJob.class);

		when(job.getAction()).thenReturn("DELETE");

		executor.execute(job);

		verify(blackboardHandler).failed(eq(job), (Throwable) anyObject());

	}

}

package eu.dnetlib.data.mdstore.modular.connector;

import java.util.Date;

public class MDStoreTransactionInfo {

	private String id;

	private Boolean refresh;

	private Date date;

	private long size;

	/**

	 * @return the date

	 */

	public Date getDate() {

		return date;

	}

	/**

	 * @param date

	 *            the date to set

	 */

	public void setDate(final Date date) {

		this.date = date;

	}

	/**

	 * @return the id

	 */

	public String getId() {

		return id;

	}

	/**

	 * @param id

	 *            the id to set

	 */

	public void setId(final String id) {

		this.id = id;

	}

	/**

	 * @return the refresh

	 */

	public Boolean getRefresh() {

		return refresh;

	}

	/**

	 * @param refresh

	 *            the refresh to set

	 */

	public void setRefresh(final Boolean refresh) {

		this.refresh = refresh;

	}

	/**

	 * @return the size

	 */

	public long getSize() {

		return size;

	}

	/**

	 * @param size

	 *            the size to set

	 */

	public void setSize(final long size) {

		this.size = size;

	}

}

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/cnr-modular-mdstore-service/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "cnr-modular-mdstore-service"}

package eu.dnetlib.data.mdstore.modular.connector;

import java.util.ArrayList;

import java.util.List;

public class MDStoreManagerInfo {

	private String mdId;

	private String currentId;

	private List<MDStoreExpiredInfo> stillUsed;

	private List<MDStoreTransactionInfo> transactions;

	public String getMdId() {

		return mdId;

	}

	public void setMdId(final String mdId) {

		this.mdId = mdId;

	}

	public String getCurrentId() {

		return currentId;

	}

	public void setCurrentId(final String currentId) {

		this.currentId = currentId;

	}

	public List<MDStoreExpiredInfo> getStillUsed() {

		return stillUsed;

	}

	public void setStillUsed(final List<MDStoreExpiredInfo> stillUsed) {

		this.stillUsed = stillUsed;

	}

	public List<MDStoreTransactionInfo> getTransactions() {

		return transactions;

	}

	public void setTransactions(final List<MDStoreTransactionInfo> transactions) {

		this.transactions = transactions;

	}

	public void addExpiredItem(final MDStoreExpiredInfo info) {

		if (stillUsed == null) {

			stillUsed = new ArrayList<MDStoreExpiredInfo>();

		}

		stillUsed.add(info);

	}

	public void addTransactionInfo(final MDStoreTransactionInfo info) {

		if (transactions == null) {

			transactions = new ArrayList<MDStoreTransactionInfo>();

		}

		transactions.add(info);

	}

}

package eu.dnetlib.data.mdstore.modular.connector;

import java.util.Date;

public class MDStoreExpiredInfo {

	private String id;

	private Date lastRead;

	/**

	 * @return the lastRead

	 */

	public Date getLastRead() {

		return lastRead;

	}

	/**

	 * @param lastRead

	 *            the lastRead to set

	 */

	public void setLastRead(final Date lastRead) {

		this.lastRead = lastRead;

	}

	/**

	 * @return the id

	 */

	public String getId() {

		return id;

	}

	/**

	 * @param id

	 *            the id to set

	 */

	public void setId(final String id) {

		this.id = id;

	}

}

package eu.dnetlib.data.mdstore.modular.connector;

public class MDStoreResults {

	private Iterable<String> records;

	private int size;

	public MDStoreResults(Iterable<String> records, int size) {

		super();

		this.records = records;

		this.size = size;

	}

	public int getSize() {

		return size;

	}

	public void setSize(int size) {

		this.size = size;

	}

	public Iterable<String> getRecords() {

		return records;

	}

	public void setRecords(Iterable<String> records) {

		this.records = records;

	}

}

package eu.dnetlib.data.mdstore.modular;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import eu.dnetlib.data.mdstore.DocumentNotFoundException;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.connector.MDStore;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.enabling.resultset.ResultSetFactory;

import org.springframework.beans.factory.annotation.Required;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

public class MDStoreRetriever {

    private MDStoreDao dao;

    private ResultSetFactory resultSetFactory;

    public W3CEndpointReference deliver(final String mdId, final String from, final String until, final String recordFilter) throws MDStoreServiceException {

        final MDStore mdStore = dao.readMDStore(mdId);

        return getResultSetFactory().createResultSet(mdStore.deliver(from, until, recordFilter));

    }

    public Iterable<String> deliver(final String format, final String layout, final String interpretation) throws MDStoreServiceException {

        try {

            return Iterables.concat(Iterables.transform(dao.listMDStores(format, layout, interpretation), new Function<String, Iterable<String>>() {

                @Override

                public Iterable<String> apply(final String mdId) {

                    try {

                        return dao.readMDStore(mdId).iterate();

                    } catch (MDStoreServiceException e) {

                        throw new RuntimeException(e);

                    }

                }

            }));

        } catch (RuntimeException e) {

            throw new MDStoreServiceException(e);

        }

    }

    public String deliverRecord(final String mdId, final String recordId) throws MDStoreServiceException, DocumentNotFoundException {

        return dao.getMDStore(mdId).getRecord(recordId);

    }

    public MDStoreDao getDao() {

        return dao;

    }

    @Required

    public void setDao(final MDStoreDao dao) {

        this.dao = dao;

    }

    @Required

    public void setResultSetFactory(final ResultSetFactory resultSetFactory) {

        this.resultSetFactory = resultSetFactory;

    }

    public ResultSetFactory getResultSetFactory() {

        return resultSetFactory;

    }

}

package eu.dnetlib.data.mdstore.modular;

import java.util.concurrent.Executor;

import java.util.concurrent.Executors;

import javax.annotation.Resource;

import org.antlr.stringtemplate.StringTemplate;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.core.io.ClassPathResource;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.enabling.tools.blackboard.BlackboardServerAction;

import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;

/**

 * The Class AbstractMDStoreAction.

 */

public abstract class AbstractMDStoreAction implements BlackboardServerAction<MDStoreActions> {

	@Resource

	private UniqueServiceLocator serviceLocator;

	/** The dao. */

	private MDStoreDao dao;

	/** The executor. */

	private final Executor executor = Executors.newCachedThreadPool();

	/** Logger */

	private static final Log log = LogFactory.getLog(AbstractMDStoreAction.class);

	private static final ClassPathResource mdstoreServiceStatusTemplate = new ClassPathResource(

			"/eu/dnetlib/data/mdstore/modular/mdstoreServiceStatusTemplate.xml.st");

	/**

	 * Execute async.

	 * 

	 * @param handler

	 *            the handler

	 * @param job

	 *            the job

	 * @throws MDStoreServiceException

	 *             the MD store service exception

	 */

	protected abstract void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws MDStoreServiceException;

	/*

	 * (non-Javadoc)

	 * 

	 * @see

	 * eu.dnetlib.enabling.tools.blackboard.BlackboardServerAction#execute(eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler,

	 * eu.dnetlib.enabling.tools.blackboard.BlackboardJob)

	 */

	@Override

	public void execute(final BlackboardServerHandler handler, final BlackboardJob job) {

		executor.execute(new Runnable() {

			@Override

			public void run() {

				try {

					handler.ongoing(job);

					executeAsync(handler, job);

				} catch (MDStoreServiceException e) {

					handler.failed(job, e);

				}

			}

		});

	}

	protected void completeWithSuccess(final BlackboardServerHandler handler, final BlackboardJob job) {

		// Don't change this synchronization rule

		synchronized (this) {

			updateMDStoreServiceProfile(job);

			handler.done(job);

		}

	}

	protected void completeWithFail(final BlackboardServerHandler handler, final BlackboardJob job, final Throwable e) {

		// Don't change this synchronization rule

		synchronized (this) {

			updateMDStoreServiceProfile(job);

			handler.failed(job, e);

		}

	}

	private void updateMDStoreServiceProfile(final BlackboardJob job) {

		final String id = job.getServiceId();

		log.info("Updating mdstore service profile status, id: " + id);

		try {

			final StringTemplate st = new StringTemplate(IOUtils.toString(mdstoreServiceStatusTemplate.getInputStream()));

			st.setAttribute("status", dao.getDBStatus());

			serviceLocator.getService(ISRegistryService.class).updateProfileNode(id, "//STATUS", st.toString());

		} catch (Exception e) {

			log.error("Error upadating profile " + id, e);

		}

	}

	/**

	 * Gets the dao.

	 * 

	 * @return the dao

	 */

	public MDStoreDao getDao() {

		return dao;

	}

	/**

	 * Sets the dao.

	 * 

	 * @param dao

	 *            the new dao

	 */

	public void setDao(final MDStoreDao dao) {

		this.dao = dao;

	}

}

package eu.dnetlib.data.mdstore.modular;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;

public class DeleteAction extends AbstractMDStoreAction {

	private UniqueServiceLocator serviceLocator;

	@Autowired

	private MDStoreTransactionManager transactionManager;

	@Override

	public void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws MDStoreServiceException {

		final String currentId = job.getParameters().get("id");

		try {

			serviceLocator.getService(ISRegistryService.class).deleteProfile(currentId);

			transactionManager.dropMDStore(currentId);

			getDao().deleteMDStore(currentId);

			completeWithSuccess(handler, job);

		} catch (Exception e) {

			throw new MDStoreServiceException("Error deleting mdstore with id " + currentId, e);

		}

	}

	public UniqueServiceLocator getServiceLocator() {

		return serviceLocator;

	}

	@Required

	public void setServiceLocator(final UniqueServiceLocator serviceLocator) {

		this.serviceLocator = serviceLocator;

	}

}

package eu.dnetlib.data.mdstore.modular;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;

public class FeedAction extends AbstractMDStoreAction {

	private static final Log log = LogFactory.getLog(FeedAction.class);

	private MDStoreFeeder feeder;

	@Override

	public void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws MDStoreServiceException {

		final String mdId = job.getParameters().get("mdId");

		if (mdId == null || mdId.isEmpty()) { throw new MDStoreServiceException("Blackboard param (mdId) is empty"); }

		final String epr = job.getParameters().get("epr");

		if (epr == null || epr.isEmpty()) { throw new MDStoreServiceException("Blackboard param (mdId) is empty"); }

		String storingType = job.getParameters().get("storingType");

		if (storingType == null || storingType.isEmpty()) {

			storingType = "REFRESH";

		}

		feeder.feed(mdId, epr, storingType, true, new FeedDoneCallback() {

			@Override

			public void call(final int size) {

				job.getParameters().put("total", "" + size);

				completeWithSuccess(handler, job);

			}

		}, new FeedFailedCallback() {

			@Override

			public void call(final Throwable e) {

				log.error("Error feeding mdstore: " + mdId, e);

				completeWithFail(handler, job, e);

			}

		});

	}

	public MDStoreFeeder getFeeder() {

		return feeder;

	}

	@Required

	public void setFeeder(final MDStoreFeeder feeder) {

		this.feeder = feeder;

	}

}

package eu.dnetlib.data.mdstore.modular;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;

public class CreateAction extends AbstractMDStoreAction {

	private MDStoreProfileCreator profileCreator;

	@Autowired

	private MDStoreTransactionManager transactionManager;

	@Override

	public void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws MDStoreServiceException {

		final String format = job.getParameters().get("format");

		final String interpretation = job.getParameters().get("interpretation");

		final String layout = job.getParameters().get("layout");

		try {

			String mdId = profileCreator.registerProfile(format, interpretation, layout);

			transactionManager.createMDStore(mdId);

			getDao().createMDStore(mdId, format, interpretation, layout);

			job.getParameters().put("id", mdId);

			completeWithSuccess(handler, job);

		} catch (ISRegistryException e) {

			throw new MDStoreServiceException(e);

		}

	}

	public MDStoreProfileCreator getProfileCreator() {

		return profileCreator;

	}

	@Required

	public void setProfileCreator(final MDStoreProfileCreator profileCreator) {

		this.profileCreator = profileCreator;

	}

}

package eu.dnetlib.data.mdstore.modular;

import javax.xml.ws.Endpoint;

import org.antlr.stringtemplate.StringTemplate;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.soap.EndpointReferenceBuilder;

public class MDStoreProfileCreator {

	/**

	 * service locator.

	 */

	private UniqueServiceLocator serviceLocator;

	/**

	 * mdstore ds template.

	 */

	private StringTemplate mdstoreDsTemplate;

	/**

	 * service endpoint.

	 */

	private Endpoint endpoint;

	/**

	 * endpoint builder.

	 */

	private EndpointReferenceBuilder<Endpoint> eprBuilder;

	public String registerProfile(String format, String interpretation, String layout) throws ISRegistryException {

		// XXX: mini hack

		StringTemplate template = new StringTemplate(mdstoreDsTemplate.getTemplate());

		template.setAttribute("serviceUri", eprBuilder.getAddress(endpoint));

		template.setAttribute("format", format);

		template.setAttribute("interpretation", interpretation);

		template.setAttribute("layout", layout);

		return serviceLocator.getService(ISRegistryService.class).registerProfile(template.toString());

	}

	public StringTemplate getMdstoreDsTemplate() {

		return mdstoreDsTemplate;

	}

	@Required

	public void setMdstoreDsTemplate(StringTemplate mdstoreDsTemplate) {

		this.mdstoreDsTemplate = mdstoreDsTemplate;

	}

	public Endpoint getEndpoint() {

		return endpoint;

	}

	@Required

	public void setEndpoint(Endpoint endpoint) {

		this.endpoint = endpoint;

	}

	public EndpointReferenceBuilder<Endpoint> getEprBuilder() {

		return eprBuilder;

	}

	@Required

	public void setEprBuilder(EndpointReferenceBuilder<Endpoint> eprBuilder) {

		this.eprBuilder = eprBuilder;

	}

	public UniqueServiceLocator getServiceLocator() {

		return serviceLocator;

	}

	@Required

	public void setServiceLocator(UniqueServiceLocator serviceLocator) {

		this.serviceLocator = serviceLocator;

	}

}

package eu.dnetlib.data.mdstore.modular;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.connector.MDStore;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;

import eu.dnetlib.miscutils.datetime.DateUtils;

public class MDStoreFeeder {

	private static final Log log = LogFactory.getLog(MDStoreFeeder.class);

	private MDStoreDao dao;

	private ResultSetClientFactory resultSetClientFactory;

	private UniqueServiceLocator serviceLocator;

	private boolean syncFeed = true;

	public void feed(final String mdId,

			final String rsEpr,

			final String storingType,

			final boolean sync,

			final FeedDoneCallback doneCallback,

			final FeedFailedCallback failCallback) throws MDStoreServiceException {

		log.info("Start feeding mdstore " + mdId + " with epr " + rsEpr);

		try {

			final boolean refresh = "REFRESH".equals(storingType);

			final MDStore mdstore = dao.startTransaction(mdId, refresh);

			final Iterable<String> records = resultSetClientFactory.getClient(rsEpr);

			if (refresh) {

				mdstore.truncate();

			}

			mdstore.feed(records, refresh);

			dao.commit(mdstore.getId(), mdId);

			int size = dao.refreshSize(mdId);

			touch(mdId, size);

			log.info("Finished feeding mdstore " + mdId + " - new size: " + size);

			doneCallback.call(size);

		} catch (Exception e) {

			log.error("Error feeding mdstore: " + mdId);

			failCallback.call(e);

		}

	}

	/**

	 * Sets the last modified date in the profile.

	 *

	 * @param mdId

	 */

	public void touch(final String mdId, final int size) {

		try {

			final String now = DateUtils.now_ISO8601();

			final String mdstoreXUpdate =

					"for $x in //RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + mdId + "']" + "return update value $x//LAST_STORAGE_DATE with '" + now

					+ "'";

			serviceLocator.getService(ISRegistryService.class).executeXUpdate(mdstoreXUpdate);

			touchSize(mdId, size);

		} catch (final Exception e) {

			throw new IllegalStateException(e);

		}

	}

	public void touchSize(final String mdId, final int size) {

		try {

			final String mdstoreNumberXUpdate =

					"for $x in //RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + mdId + "']" + "return update value $x//NUMBER_OF_RECORDS with '" + size

					+ "'";

			serviceLocator.getService(ISRegistryService.class).executeXUpdate(mdstoreNumberXUpdate);

		} catch (final Exception e) {

			throw new IllegalStateException(e);

		}

	}

	public MDStoreDao getDao() {

		return dao;

	}

	@Required

	public void setDao(final MDStoreDao dao) {

		this.dao = dao;

	}

	public ResultSetClientFactory getResultSetClientFactory() {

		return resultSetClientFactory;

	}

	@Required

	public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {

		this.resultSetClientFactory = resultSetClientFactory;

	}

	public boolean isSyncFeed() {

		return syncFeed;

	}

	public void setSyncFeed(final boolean syncFeed) {

		this.syncFeed = syncFeed;

	}

	public UniqueServiceLocator getServiceLocator() {

		return serviceLocator;

	}

	@Required

	public void setServiceLocator(UniqueServiceLocator serviceLocator) {

		this.serviceLocator = serviceLocator;

	}

}

package eu.dnetlib.data.mdstore.modular;

import java.util.List;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import com.google.common.base.Function;

import com.google.common.collect.Lists;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.data.mdstore.DocumentNotFoundException;

import eu.dnetlib.data.mdstore.MDStoreService;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.enabling.resultset.IterableResultSetFactory;

import eu.dnetlib.enabling.tools.AbstractBaseService;

import eu.dnetlib.enabling.tools.blackboard.NotificationHandler;

public class ModularMDStoreService extends AbstractBaseService implements MDStoreService {

	/**

	 * notification handler.

	 */

	private NotificationHandler notificationHandler;

	private MDStoreFeeder feeder;

	private MDStoreRetriever retriever;

	private IterableResultSetFactory iterableResultSetFactory;

	@Override

	public W3CEndpointReference deliverMDRecords(final String mdId, final String from, final String until, final String recordFilter)

			throws MDStoreServiceException {

		return retriever.deliver(mdId, from, until, recordFilter);

	}

	@Override

	public W3CEndpointReference bulkDeliverMDRecords(final String format, final String layout, final String interpretation) throws MDStoreServiceException {

		return getIterableResultSetFactory().createIterableResultSet(retriever.deliver(format, layout, interpretation));

	}

	@Override

	public String deliverRecord(final String mdId, final String recordId) throws MDStoreServiceException, DocumentNotFoundException {

		return retriever.deliverRecord(mdId, recordId);

	}

	@Override

	public List<String> getListOfMDStores() throws MDStoreServiceException {

		return (Lists.transform(retriever.getDao().listMDStores(), new Function<MDStoreDescription, String>() {

            @Override

            public String apply(MDStoreDescription input) {

                return input.getId();

            }

        }));

	}

	@Override

	public List<String> listMDStores(final String format, final String layout, final String interpretation) throws MDStoreServiceException {

		return retriever.getDao().listMDStores(format, layout, interpretation);

	}

	@Override

	public void notify(final String subscriptionId, final String topic, final String isId, final String message) {

		getNotificationHandler().notified(subscriptionId, topic, isId, message);

	}

	@Override

	public boolean storeMDRecordsFromRS(final String mdId, final String rsId, final String storingType) throws MDStoreServiceException {

		throw new MDStoreServiceException("not implemented, use the Blackboard asynchronous equivalent");

	}

	public NotificationHandler getNotificationHandler() {

		return notificationHandler;

	}

	@Required

	public void setNotificationHandler(final NotificationHandler notificationHandler) {

		this.notificationHandler = notificationHandler;

	}

	public MDStoreFeeder getFeeder() {

		return feeder;

	}

	public void setFeeder(final MDStoreFeeder feeder) {

		this.feeder = feeder;

	}

	public MDStoreRetriever getRetriever() {

		return retriever;

	}

	public void setRetriever(final MDStoreRetriever retriever) {

		this.retriever = retriever;

	}

	public IterableResultSetFactory getIterableResultSetFactory() {

		return iterableResultSetFactory;

	}

	@Required

	public void setIterableResultSetFactory(final IterableResultSetFactory iterableResultSetFactory) {

		this.iterableResultSetFactory = iterableResultSetFactory;

	}

}

package eu.dnetlib.data.mdstore.modular;

public class MDStoreDescription {

	private String id;

	private String format;

	private String layout;

	private String interpretation;

	private int size;

	private boolean indexed;

	public MDStoreDescription(final String id, final String format, final String layout, final String interpretation, final int size, final boolean indexed) {

		super();

		this.id = id;

		this.format = format;

		this.layout = layout;

		this.interpretation = interpretation;

		this.size = size;

		this.indexed = indexed;

	}

    public MDStoreDescription() {

    }

    public String getId() {

		return id;

	}

	public void setId(final String id) {

		this.id = id;

	}

	public String getFormat() {

		return format;

	}

	public void setFormat(final String format) {

		this.format = format;

	}