D-Net

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws"

	xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing"

	xmlns:p="http://www.springframework.org/schema/p" xmlns:http="http://cxf.apache.org/transports/http/configuration"

	xmlns:t="http://dnetlib.eu/springbeans/t" xmlns:template="http://dnetlib.eu/springbeans/template"

	xmlns:util="http://www.springframework.org/schema/util"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd

    	http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd

        http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd

        http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd

        http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd

        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd

        http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">

	<!-- endpoints -->

	<jaxws:endpoint id="oaiStoreServiceEndpoint"

		implementor="#oaiStoreService" implementorClass="eu.dnetlib.data.oai.store.OAIStoreService"

		address="/oaistore" />

	<template:instance name="serviceRegistrationManager"

		t:serviceRegistrationManagerClass="eu.dnetlib.enabling.tools.registration.ValidatingServiceRegistrationManagerImpl"

		t:name="oaiStoreServiceRegistrationManager" t:service="oaiStoreService"

		t:endpoint="oaiStoreServiceEndpoint" t:jobScheduler="jobScheduler"

		t:serviceRegistrator="blackboardServiceRegistrator" />

	<bean id="oaiStoreService" class="eu.dnetlib.data.oai.store.OAIStoreServiceImpl"

		init-method="start" destroy-method="stop" p:notificationHandler-ref="oaiStoreNotificationHandler" />

	<bean id="oaiStoreNotificationHandler"

		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerExecutorNotificationHandler"

		p:blackboardExecutor-ref="oaiStoreBlackboardExecutor" />

	<bean id="oaiStoreBlackboardExecutor"

		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerActionExecutor"

		p:blackboardHandler-ref="blackboardHandler"

		p:actionType="eu.dnetlib.data.oai.store.actions.OAIStoreActions"

		p:incomplete="false">

		<property name="actionMap">

			<map>

				<entry key="SYNC">

					<bean class="eu.dnetlib.data.oai.store.actions.SyncAction" />

				</entry>

				<entry key="COUNT_SETS">

					<bean class="eu.dnetlib.data.oai.store.actions.CountSetsAction"/>

				</entry>

				<entry key="REFRESH_CONFIG">

					<bean class="eu.dnetlib.data.oai.store.actions.RefreshConfigAction" />

				</entry>

				<entry key="ENSURE_INDEXES">

					<bean class="eu.dnetlib.data.oai.store.actions.EnsureIndexesAction" />

				</entry>

				<entry key="CREATE_STORE">

					<bean class="eu.dnetlib.data.oai.store.actions.CreateStoreAction" />

				</entry>

				<entry key="CREATE_OAI_INDEX">

					<bean class="eu.dnetlib.data.oai.store.actions.CreateOAIIndexAction" />

				</entry>

				<entry key="DROP_STORE">

					<bean class="eu.dnetlib.data.oai.store.actions.DropStoreAction" />

				</entry>

			</map>

		</property>

	</bean>

	<bean id="synchronizer" class="eu.dnetlib.data.oai.store.sync.OAIStoreSynchronizer" />

	<bean id="oaiHelper" class="eu.dnetlib.data.oai.store.conf.OAIHelper"/>

	<bean id="oaiSetsCounter" class="eu.dnetlib.data.oai.store.conf.OAISetsCounter" />

	<bean id="mongoCacheHelper" class="eu.dnetlib.data.oai.store.mongo.MongoPublisherCacheHelper"/>

</beans>

services.publisher.oai.host=localhost

services.publisher.oai.port=27017

#services.publisher.oai.db=oaistore

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

       xmlns:p="http://www.springframework.org/schema/p" xmlns="http://www.springframework.org/schema/beans"

       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-4.0.xsd">

	<bean id="mongodbPublisherStoreDao" class="eu.dnetlib.data.oai.store.mongo.MongoPublisherStoreDAO"

		p:metadataCollection="metadata" />

	<bean id="publisherMongoClient" class="com.mongodb.MongoClient">

		<constructor-arg index="0" type="com.mongodb.ServerAddress">

			<bean class="com.mongodb.ServerAddress">

				<constructor-arg index="0"

					value="${services.publisher.oai.host}" />

				<constructor-arg index="1"

					value="${services.publisher.oai.port}" />

			</bean>

		</constructor-arg>

	</bean>

	<bean id="recordChangeDetector" class="eu.dnetlib.data.oai.store.DummyRecordChangeDetector" />

	<bean id="metadataExtractor" class="eu.dnetlib.data.oai.store.mongo.MetadataExtractor" />

	<bean id="recordInfoGenerator" class="eu.dnetlib.data.oai.store.mongo.RecordInfoGenerator" />

	<bean id="mongoSetCollection" class="eu.dnetlib.data.oai.store.sets.MongoSetCollection" />

	<bean id="provenanceExtractor" class="eu.dnetlib.data.oai.store.mongo.ProvenanceExtractor" />

	<bean id="theOAICacheManager" class="org.springframework.cache.ehcache.EhCacheCacheManager"

        p:cache-manager-ref="oaiStoreCacheManager" />

	<bean id="oaiStoreCacheManager"

		class="org.springframework.cache.ehcache.EhCacheManagerFactoryBean"

		p:cacheManagerName="oaiStoreCacheManger"

		p:configLocation="classpath:/eu/dnetlib/data/oai/store/mongo/cache/ehcache.xml" />

</beans>

<ehcache>

	<!-- Sets the path to the directory where cache .data files are created. 

		If the path is a Java System Property it is replaced by its value in the 

		running VM. The following properties are translated: user.home - User's home 

		directory user.dir - User's current working directory java.io.tmpdir - Default 

		temp file path -->

	<!--<diskStore path="java.io.tmpdir" />-->

	<!--Default Cache configuration. These will applied to caches programmatically 

		created through the CacheManager. The following attributes are required for 

		defaultCache: maxInMemory - Sets the maximum number of objects that will 

		be created in memory eternal - Sets whether elements are eternal. If eternal, 

		timeouts are ignored and the element is never expired. timeToIdleSeconds 

		- Sets the time to idle for an element before it expires. i.e. The maximum 

		amount of time between accesses before an element expires Is only used if 

		the element is not eternal. Optional attribute. A value of 0 means that an 

		Element can idle for infinity timeToLiveSeconds - Sets the time to live for 

		an element before it expires. i.e. The maximum time between creation time 

		and when an element expires. Is only used if the element is not eternal. 

		overflowToDisk - Sets whether elements can overflow to disk when the in-memory 

		cache has reached the maxInMemory limit. -->

	<!-- <defaultCache maxElementsInMemory="1000" overflowToDisk="true" -->

	<!-- eternal="true" diskPersistent="false" timeToIdleSeconds="0" -->

	<!-- timeToLiveSeconds="0" memoryStoreEvictionPolicy="LRU" -->

	<!-- diskExpiryThreadIntervalSeconds="120" diskSpoolBufferSizeMB="5" /> -->

	<cache name="oaistores" maxElementsInMemory="100" eternal="false"

		timeToIdleSeconds="1800" timeToLiveSeconds="1800" overflowToDisk="false" />

	<cache name="oaistoresById" maxElementsInMemory="100"

		eternal="false" timeToIdleSeconds="1800" timeToLiveSeconds="1800"

		overflowToDisk="false" />

</ehcache>

<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.0">

    <xsl:template match="//*[local-name()='metadata']">

        <metadata>

            <dmf>

                <xsl:apply-templates/>

            </dmf>

        </metadata>

    </xsl:template>

    <xsl:template match="@*|node()">

        <xsl:copy>

            <xsl:apply-templates select="@*|node()"/>

        </xsl:copy>

    </xsl:template>

</xsl:stylesheet>

package eu.dnetlib.data.oai.store;

import javax.annotation.Resource;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import eu.dnetlib.data.oai.store.conf.OAIHelper;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.data.information.oai.publisher.info.MDFInfo;

import eu.dnetlib.data.oai.store.conf.OAISetsCounter;

import eu.dnetlib.data.oai.store.mongo.MongoPublisherStore;

import eu.dnetlib.data.oai.store.mongo.MongoPublisherStoreDAO;

import eu.dnetlib.data.oai.store.sync.OAIStoreSynchronizer;

import eu.dnetlib.enabling.resultset.client.IterableResultSetClient;

import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;

import eu.dnetlib.enabling.tools.AbstractBaseService;

import eu.dnetlib.enabling.tools.blackboard.NotificationHandler;

public class OAIStoreServiceImpl extends AbstractBaseService implements OAIStoreService {

	/**

	 * notification handler.

	 */

	private NotificationHandler notificationHandler;

	@Autowired

	private MongoPublisherStoreDAO mongoPublisherStoreDAO;

	@Resource

	private OAIStoreSynchronizer synchronizer;

	@Resource(name = "oaiSetsCounter")

	private OAISetsCounter setsCounter;

	@Autowired

	private OAIHelper oaiHelper;

	@Resource

	private ResultSetClientFactory resultSetClientFactory;

	@Override

	public void feed(final W3CEndpointReference recordsEpr, final MDFInfo mdfInfo, final String dbName) {

		this.feed(recordsEpr, mdfInfo, "", dbName);

	}

	@Override

	public void feed(final W3CEndpointReference recordsEpr, final MDFInfo mdfInfo, final String recordSource, final String dbName) {

		IterableResultSetClient rsClient = resultSetClientFactory.getClient(recordsEpr);

		this.synchronizer.synchronize(rsClient, mdfInfo, recordSource, dbName, false, null, null);

	}

	@Override

	public void updateSetCounts(final String dbName) {

		this.setsCounter.updateCounts("", dbName, null, null);

	}

	@Override

	public void updateSetCounts(final MDFInfo mdfInfo, final String dbName) {

		this.setsCounter.updateCounts(mdfInfo, dbName, null, null);

	}

	@Override

	public void refreshConfiguration(final String dbName) {

		oaiHelper.loadConfiguration(dbName);

	}

	@Override

	public void ensureIndexes(final String dbName) {

		this.mongoPublisherStoreDAO.ensureIndex(dbName);

	}

	@Override

	public void ensureIndexes(final MDFInfo mdfInfo, final String dbName) {

		MongoPublisherStore s = this.mongoPublisherStoreDAO.getStore(mdfInfo.getSourceFormatName(), mdfInfo.getSourceFormatInterpretation(),

				mdfInfo.getSourceFormatLayout(), dbName);

		s.ensureIndices();

	}

	@Override

	public void notify(final String subscriptionId, final String topic, final String isId, final String message) {

		getNotificationHandler().notified(subscriptionId, topic, isId, message);

	}

	public NotificationHandler getNotificationHandler() {

		return notificationHandler;

	}

	@Required

	public void setNotificationHandler(final NotificationHandler notificationHandler) {

		this.notificationHandler = notificationHandler;

	}

	public OAIStoreSynchronizer getSynchronizer() {

		return synchronizer;

	}

	public void setSynchronizer(final OAIStoreSynchronizer synchronizer) {

		this.synchronizer = synchronizer;

	}

	public OAISetsCounter getSetsCounter() {

		return setsCounter;

	}

	public void setSetsCounter(final OAISetsCounter setsCounter) {

		this.setsCounter = setsCounter;

	}

	public OAIHelper getOaiHelper() {

		return oaiHelper;

	}

	public void setOaiHelper(final OAIHelper oaiHelper) {

		this.oaiHelper = oaiHelper;

	}

	public MongoPublisherStoreDAO getMongoPublisherStoreDAO() {

		return mongoPublisherStoreDAO;

	}

	public void setMongoPublisherStoreDAO(final MongoPublisherStoreDAO mongoPublisherStoreDAO) {

		this.mongoPublisherStoreDAO = mongoPublisherStoreDAO;

	}

	public ResultSetClientFactory getResultSetClientFactory() {

		return resultSetClientFactory;

	}

	public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {

		this.resultSetClientFactory = resultSetClientFactory;

	}

}

package eu.dnetlib.data.oai.store;

/**

 * This is a basic dummy implementation that always returns true, regardless the actual content of the records to compare.

 * 

 * @author alessia

 * 

 */

public class DummyRecordChangeDetector implements RecordChangeDetector {

	/**

	 * 

	 * {@inheritDoc}

	 * 

	 * @see eu.dnetlib.data.oai.store.RecordChangeDetector#differs(java.lang.String, java.lang.String)

	 */

	@Override

	public boolean differs(final String record1, final String record2) {

		return true;

	}

}

package eu.dnetlib.data.oai.store;

import java.util.List;

import eu.dnetlib.data.information.oai.publisher.PublisherField;

import eu.dnetlib.data.information.oai.publisher.info.RecordInfo;

import eu.dnetlib.miscutils.functional.UnaryFunction;

/**

 * Interface for store used by the publisher. Different back-ends require different implementation.

 * 

 * @author alessia

 * 

 * @param <T>

 *            class of the object that can be used to iterate results of a bulk request of records from the store.

 */

public interface PublisherStore<T extends Cursor> {

	/**

	 * Returns the identifier of this store.

	 * 

	 * @return a String that identifies this store

	 */

	String getId();

	/**

	 * Gets the metadata format of this store.

	 * 

	 * @return String

	 */

	String getMetadataFormat();

	/**

	 * Gets the interpretation of this store.

	 * 

	 * @return String

	 */

	String getInterpretation();

	/**

	 * Gets the layout of this store.

	 * 

	 * @return String

	 */

	String getLayout();

	/**

	 * Retrieves the record with the given identifier from this store.

	 * 

	 * @param recordId

	 *            identifier of the record to retrieve

	 * @return a RecordInfo instance representing the XML record

	 */

	RecordInfo getRecord(final String recordId);

	/**

	 * Retrieves the record with the given identifier from this store and apply the given unary function before delivering.

	 * 

	 * @param recordId

	 *            identifier of the record to retrieve

	 * @param unaryFunction

	 *            mapping from String to String

	 * @return a RecordInfo instance representing the XML record

	 */

	RecordInfo getRecord(final String recordId, final UnaryFunction<String, String> unaryFunction);

	/**

	 * Retrieves the records matching the given query string.

	 * 

	 * @param queryString

	 *            search criterion

	 * @param bodyIncluded

	 *            false to get only identifiers, true to also get the body of records

	 * @param limit

	 *            max number of records to return. 0 means no limit.

	 * @return an instance of T that can be used to iterate over the results.

	 */

	T getRecords(final String queryString, final boolean bodyIncluded, final int limit);

	/**

	 * Retrieves the records matching the given query string and apply the given unary function before delivering.

	 * 

	 * @param queryString

	 *            search criterion

	 * @param unaryFunction

	 *            mapping from String to String

	 * @param bodyIncluded

	 *            false to get only identifiers, true to also get the body of records

	 * @param limit

	 *            max number of records to return. 0 means no limit.

	 * @return an instance of T that can be used to iterate over the results.

	 */

	T getRecords(final String queryString, final UnaryFunction<String, String> unaryFunction, final boolean bodyIncluded, final int limit);

	/**

	 * Gets informaton about the indices available on this store.

	 * 

	 * @return a List with information about the indices of this store.

	 */

	List<PublisherField> getIndices();

	/**

	 * Ensures the indices on this store.

	 */

	void ensureIndices();

	/**

	 * Feeds the store with the given records.

	 * 

	 * @param records

	 *            XML records to store.

	 * @param source

	 *            String that identifies the record source.

	 * @return the number of stored records.

	 */

	int feed(final Iterable<String> records, final String source);

	/**

	 * Drops the content of the store.

	 * 

	 */

	void drop();

	/**

	 * Drops the elements matching the query from the store.

	 * 

	 * @param queryString

	 */

	void drop(final String queryString);

	/**

	 * Counts the number of records in this store.

	 * 

	 * @return the size of the store.

	 */

	int count();

	/**

	 * Counts the number of records in this store matching the query.

	 * 

	 * @param queryString

	 *            query

	 * @return the size of the store.

	 */

	int count(final String queryString);

}

package eu.dnetlib.data.oai.store.parser;

import java.io.StringReader;

import java.util.List;

import java.util.Map.Entry;

import com.google.common.base.Function;

import com.google.common.collect.ArrayListMultimap;

import com.google.common.collect.Iterables;

import com.google.common.collect.Multimap;

import eu.dnetlib.data.information.oai.publisher.PublisherField;

import eu.dnetlib.data.information.oai.publisher.conf.OAIConfigurationExistReader;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

/**

 * An instance of this class can parse an XML record and extract the information needed to store the record in a publisher store.

 * 

 * 

 * @author alessia

 * 

 */

public class PublisherRecordParser {

	private static final Log log = LogFactory.getLog(PublisherRecordParser.class); // NOPMD by marko on 11/24/08 5:02 PM

	/**

	 * List of the indices of the target store.

	 */

	private List<PublisherField> storeIndices;

	private final SAXReader saxReader = new SAXReader();

	/**

	 * Parses the record and returns a map where a key is the name of an index, the value is the value in the record at the xpath specificed

	 * in this.storeIndices.

	 * 

	 * @param record

	 *            the XML string to parse.

	 * @return a Multimap describing the values to be indexed for this record.

	 */

	@SuppressWarnings({ "unchecked", "rawtypes" })

	public Multimap<String, String> parseRecord(final String record, final String source) {

		Multimap<String, String> recordProps = ArrayListMultimap.create();

		try {

			Document doc = this.saxReader.read(new StringReader(record));

			if(StringUtils.isNotBlank(source)) recordProps.put(OAIConfigurationExistReader.SET_FIELD, source);

			for (PublisherField field : this.storeIndices) {

				for (Entry<String, String> indexEntry : field.getSources().entries()) {

					// each xpath can return a list of nodes or strings, depending on the xpath

					List xPathResult = doc.selectNodes(indexEntry.getValue());

					if ((xPathResult != null) && !xPathResult.isEmpty()) {

						if (containsStrings(xPathResult)) {

							recordProps.putAll(field.getFieldName(), xPathResult);

						} else {

							if (containsNodes(xPathResult)) {

								recordProps.putAll(field.getFieldName(), Iterables.transform(xPathResult, new Function<Object, String>() {

									@Override

									public String apply(final Object obj) {

										if (obj == null) return "";

										Node node = (Node) obj;

										return node.getText();

									}

								}));

							}

						}

					}

				}

			}

		} catch (DocumentException e) {

			log.fatal("Can't parse record");

			recordProps = null;

		}

		return recordProps;

	}

	@SuppressWarnings("rawtypes")

	private boolean containsStrings(final List objects) {

		Object first = objects.get(0);

		return first instanceof String;

	}

	@SuppressWarnings("rawtypes")

	private boolean containsNodes(final List objects) {

		Object first = objects.get(0);

		return first instanceof Node;

	}

	public List<PublisherField> getStoreIndices() {

		return storeIndices;

	}

	public void setStoreIndices(final List<PublisherField> storeIndices) {

		this.storeIndices = storeIndices;

	}

	public SAXReader getSaxReader() {

		return saxReader;

	}

	public PublisherRecordParser(final List<PublisherField> storeIndices) {

		super();

		this.storeIndices = storeIndices;

	}

	public PublisherRecordParser() {

		super();

		// TODO Auto-generated constructor stub

	}

}

package eu.dnetlib.data.oai.store.sync;

import java.util.concurrent.Callable;

import javax.annotation.Resource;

import eu.dnetlib.data.information.oai.publisher.OaiPublisherException;

import eu.dnetlib.data.information.oai.publisher.OaiPublisherRuntimeException;

import eu.dnetlib.data.information.oai.publisher.conf.OAIConfigurationExistReader;

import eu.dnetlib.data.information.oai.publisher.info.MDFInfo;

import eu.dnetlib.data.oai.store.mongo.MongoPublisherStore;

import eu.dnetlib.data.oai.store.mongo.MongoPublisherStoreDAO;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

public class OAIStoreSynchronizer {

	private static final Log log = LogFactory.getLog(OAIStoreSynchronizer.class); // NOPMD by marko on 11/24/08 5:02 PM

	/**

	 * OAI Publisher configuration.

	 */

	@Resource

	private OAIConfigurationExistReader configuration;

	@Autowired

	private MongoPublisherStoreDAO mongoPublisherStoreDAO;

	public void synchronize(final Iterable<String> records,

			final MDFInfo sourceMetadataFormat,

			final String recordSource,

			final String dbName,

			final boolean alwaysNewRecord,

			final Callable<?> callback,

			final Callable<?> failCallback) {

		try {

			log.info("Synchronizing content for oai on db " + dbName + " for metadata format: " + sourceMetadataFormat+" . Record source: "+recordSource);

			MongoPublisherStore store = this.getStore(sourceMetadataFormat, dbName, alwaysNewRecord);

			int count = store.feed(records, recordSource);

			log.info("Content synchronized: store " + store.getId() + " fed with " + count + " records");

			executeCallback(callback);

		} catch (Exception e) {

			log.error(e);

			executeCallback(failCallback);

		}

	}

	/**

	 * Gets the OAI store for the given source metadata format. If the store does not exists, then a new one is created.

	 * 

	 * @param sourceMetadataFormat

	 *            MDFInfo about the metadata format of the store to get

	 * @return a MongoPublisherStore instance

	 */

	private MongoPublisherStore getStore(final MDFInfo sourceMetadataFormat, final String dbName, final boolean alwaysNewRecord) {

		this.mongoPublisherStoreDAO.setAlwaysNewRecord(alwaysNewRecord);

		MongoPublisherStore store = this.mongoPublisherStoreDAO.getStore(sourceMetadataFormat.getSourceFormatName(),

				sourceMetadataFormat.getSourceFormatInterpretation(), sourceMetadataFormat.getSourceFormatLayout(), dbName);

		if (store == null) {

			log.debug("Creating store for metadata format: \n" + sourceMetadataFormat + " in db: " + dbName);

			try {

				store = this.mongoPublisherStoreDAO.createStore(sourceMetadataFormat.getSourceFormatName(), sourceMetadataFormat.getSourceFormatInterpretation(),

						sourceMetadataFormat.getSourceFormatLayout(), dbName);

				log.debug("Created store with id: " + store.getId());

			} catch (OaiPublisherException e) {

				throw new OaiPublisherRuntimeException(e);

			}

		}

		return store;

	}

	protected void executeCallback(final Callable<?> callback) {

		if (callback != null) {

			try {

				callback.call();

			} catch (Exception e) {

				log.error("Error executing callback", e);

			}

		}

	}

	public OAIConfigurationExistReader getConfiguration() {

		return configuration;

	}

	public void setConfiguration(final OAIConfigurationExistReader configuration) {

		this.configuration = configuration;

	}

}

package eu.dnetlib.data.oai.store;

import eu.dnetlib.data.information.oai.publisher.info.RecordInfo;

public interface Cursor extends Iterable<RecordInfo> {

	int count();

	boolean isBodyIncluded();

	void setBodyIncluded(boolean bodyIncluded);

}

package eu.dnetlib.data.oai.store;

public interface RecordChangeDetector {

	/**

	 * Checks if the two records have differences based on logics that vary on the actual implementor class.

	 * 

	 * @param record1

	 *            first record to compare

	 * @param record2

	 *            second record to compare

	 * @return true if the two records differ based on the implementor's logics

	 */

	boolean differs(final String record1, final String record2);

}

package eu.dnetlib.data.oai.store;

import java.util.List;

import eu.dnetlib.data.information.oai.publisher.OaiPublisherException;

public interface PublisherStoreDAO<X extends PublisherStore<T>, T extends Cursor> {

	/**

	 * Lists all PublisherStore.

	 * 

	 * @param dbName

	 *            name of the target database

	 * 

	 * @return a List of PublisherStore instances.

	 */

	List<X> listPublisherStores(final String dbName);

	/**

	 * Gets the store with the given identifier.

	 * 

	 * @param dbName

	 *            name of the target database

	 * 

	 * @param storeId

	 *            identifier of the store to retrieve

	 * @return a PublisherStore instance or null if there is no store with the given id.

	 */

	X getStore(final String storeId, final String dbName);

	/**

	 * Gets the store with the given properties.

	 * 

	 * @param mdfName

	 *            name of the metadata format

	 * @param mdfInterpretation

	 *            name of the metadata interpretation

	 * @param mdfLayout

	 *            name of the metadata layout

	 * @param dbName

	 *            name of the target database

	 * @return a PublisherStore instance or null if there is no store with the given properties.

	 */

	X getStore(final String mdfName, final String mdfInterpretation, final String mdfLayout, final String dbName);

	/**

	 * Gets the store to be used as source to deliver records with the given metadata prefix.

	 * 

	 * @param targetMetadataPrefix

	 *            prefix of the metadata format deliverable through this store

	 * @param dbName

	 *            name of the target database

	 * @return a PublisherStore instance or null if there is no store serving the given metadata prefix.

	 */

	X getStoreFor(final String targetMetadataPrefix, final String dbName);

	/**

	 * Create a PublisherStore with the given properties.

	 * 

	 * @param mdfName

	 *            name of the metadata format

	 * @param mdfInterpretation

	 *            name of the metadata interpretation

	 * @param mdfLayout

	 *            name of the metadata layout

	 * @param dbName

	 *            name of the target database

	 * @return a PublisherStore instance whose identifier is automatically generated.

	 * @throws OaiPublisherException

	 *             if there is already another PublisherStore with the given metadata format name, layout and interpretation.

	 */

	X createStore(final String mdfName, final String mdfInterpretation, final String mdfLayout, final String dbName) throws OaiPublisherException;

	/**

	 * Deletes the store with the given identifier.

	 * 

	 * @param storeId

	 *            id of the store to delete

	 * @param dbName

	 *            name of the target database

	 * @return true if the store was deleted successfully, false otherwise (e.g., a store with the given id does not exist).

	 */

	boolean deleteStore(final String storeId, final String dbName);

	/**

	 * Deletes from the store with the given identifier the records belonging to the given set.

	 * 

	 * @param storeId

	 *            id of the store to delete

	 * @param dbName

	 *            name of the target database

	 * @param set

	 *            name of the set

	 * @return true if the records were deleted successfully, false otherwise (e.g., a store with the given id does not exist).

	 */

	boolean deleteFromStore(final String storeId, final String dbName, final String set);

	/**

	 * Deletes from the store with the given identifier the records belonging to the given set.

	 * 

	 * @param mdfName

	 *            name of the metadata format

	 * @param mdfInterpretation

	 *            name of the metadata interpretation

	 * @param mdfLayout

	 *            name of the metadata layout

	 * @param dbName

	 *            name of the target database

	 * @param set

	 *            name of the set

	 * @return true if the records were deleted successfully, false otherwise (e.g., a store with the given id does not exist).

	 */

	boolean deleteFromStore(final String mdfName, final String mdfInterpretation, final String mdfLayout, final String dbName, final String set);

	/**

	 * Deletes the store with the given properties.

	 * 

	 * @param mdfName

	 *            name of the metadata format

	 * @param mdfInterpretation

	 *            name of the metadata interpretation

	 * @param mdfLayout

	 *            name of the metadata layout

	 * @param dbName

	 *            name of the target database

	 * @return true if the store was deleted successfully, false otherwise (e.g., a store with the given properties does not exist).

	 */

	boolean deleteStore(final String mdfName, final String mdfInterpretation, final String mdfLayout, final String dbName);

}

package eu.dnetlib.data.oai.store.sets;

import java.text.Normalizer;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.mongodb.BasicDBObject;

import com.mongodb.BasicDBObjectBuilder;

import com.mongodb.DBObject;

import com.mongodb.MongoClient;

import com.mongodb.client.FindIterable;

import com.mongodb.client.MongoCollection;

import com.mongodb.client.model.Filters;

import com.mongodb.client.model.FindOneAndReplaceOptions;

import com.mongodb.client.model.FindOneAndUpdateOptions;

import com.mongodb.client.model.IndexOptions;

import eu.dnetlib.data.information.oai.publisher.info.SetInfo;

import eu.dnetlib.data.information.oai.sets.SetCollection;

import org.apache.commons.lang.StringEscapeUtils;

import org.apache.commons.lang.StringUtils;

import org.bson.conversions.Bson;

import org.springframework.beans.factory.annotation.Autowired;

public class MongoSetCollection implements SetCollection {

	public static String DEFAULT_SET = "OTHER";

	@Autowired

	private MongoClient publisherMongoClient;

	private String setCollection = "sets";

	private String setCountCollection = "setsCount";

	public void ensureIndexes(final String dbName) {

		this.ensureIndexesOnSets(dbName);

		this.ensureIndexesOnCount(dbName);

	}

	@Override

	public List<SetInfo> getAllSets(final boolean enabledOnly, final String dbName) {

		FindIterable<DBObject> iter = null;

		if (!enabledOnly) {

			iter = this.getSetsCollection(dbName).find();

		} else {

			Bson where = Filters.eq("enabled", true);

			iter = this.getSetsCollection(dbName).find(where);

		}

		return Lists.newArrayList(Iterables.transform(iter, new Function<DBObject, SetInfo>() {

			@Override

			public SetInfo apply(final DBObject dbObject) {

				return getSetFromDBObject(dbObject);

			}

		}));

	}

	@Override

	public boolean containSet(final String set, final String dbName) {

		Bson query = Filters.eq("spec", set);

		return this.getSetsCollection(dbName).count(query) != 0;

	}

	@Override

	public boolean containEnabledSet(final String set, final String publisherDBName) {

		Bson query = Filters.and(Filters.eq("spec", set), Filters.eq("enabled", true));

		return this.getSetsCollection(publisherDBName).count(query) != 0;

	}

	@Override

	public String getSetQuery(final String set, final String dbName) {

		Bson query = Filters.eq("spec", set);

		BasicDBObject returnField = new BasicDBObject("query", 1);

		DBObject obj = this.getSetsCollection(dbName).find(query).projection(returnField).first();

		return (String) obj.get("query");

	}

	@Override

	public int count(final String setSpec, final String mdPrefix, final String dbName) {

		Bson query = Filters.and(Filters.eq("spec", setSpec), Filters.eq("mdPrefix", mdPrefix));

		BasicDBObject returnField = new BasicDBObject("count", 1);

		DBObject obj = this.getSetsCountCollection(dbName).find(query).projection(returnField).first();

		if (obj == null) return 0;

		return (Integer) obj.get("count");

	}

	public void updateCounts(final String setSpec, final String mdPrefix, final int count, final String dbName) {

		BasicDBObject countUpdate = new BasicDBObject("$set", new BasicDBObject("count", count));

		Bson query = Filters.and(Filters.eq("spec", setSpec), Filters.eq("mdPrefix", mdPrefix));

		this.getSetsCountCollection(dbName).findOneAndUpdate(query, countUpdate, new FindOneAndUpdateOptions().upsert(true));

	}

	public void upsertSet(final SetInfo setInfo, final boolean fromConfiguration, final String dbName) {

		DBObject obj = this.getObjectFromSet(setInfo);

		obj.put("fromConfiguration", fromConfiguration);

		//this.getSetsCollection(dbName).update(new BasicDBObject("spec", setInfo.getSetSpec()), obj, true, false);

		this.getSetsCollection(dbName).findOneAndReplace(Filters.eq("spec", setInfo.getSetSpec()), obj, new FindOneAndReplaceOptions().upsert(true));

	}

	public String normalizeSetSpec(final String setName) {

		String s = StringEscapeUtils.unescapeXml(setName);

		s = Normalizer.normalize(s, Normalizer.Form.NFD);

		// replace spaces with underscores

		s = s.replaceAll(" ", "_");

		// remove tilde, dots... over letters

		s = s.replaceAll("[\\p{InCombiningDiacriticalMarks}&&[^-_]]", "");

		// change punctuation into an underscore

		s = s.replaceAll("[\\p{Punct}&&[^-_]]", "_");

		// remove all non-word characters

		s = s.replaceAll("[\\W&&[^-_]]", "");

		// Avoiding set '___' generated when we have "strange" set names such as those in cyrillic/ukrain

		// strips _ from the beginning and the end

		String stripped = StringUtils.strip(s, "_ ");

		if (StringUtils.isBlank(stripped)) {

			stripped = DEFAULT_SET;

		}

		return stripped;

	}

	public List<SetInfo> getConfiguredSets(final String dbName) {

		Bson query = Filters.eq("fromConfiguration", true);

		return this.findSets(query, dbName);

	}

	public List<SetInfo> getSetsFromData(final String dbName) {

		Bson query = Filters.eq("fromConfiguration", false);

		return this.findSets(query, dbName);

	}

	public void dropOAISets(final String dbName) {

		this.getSetsCountCollection(dbName).drop();

		this.getSetsCollection(dbName).drop();

	}

	public void dropSet(final String dbName, final String setSpec) {

		Bson query = Filters.eq("spec", setSpec);

		this.getSetsCollection(dbName).deleteMany(query);

		this.getSetsCountCollection(dbName).deleteMany(query);

	}

	public void dropConfigurationSets(final String dbName) {

		this.getSetsCollection(dbName).deleteMany(Filters.eq("fromConfiguration", true));

	}

	protected List<SetInfo> findSets(final Bson query, final String dbName) {

		final FindIterable<DBObject> sets = this.getSetsCollection(dbName).find(query);

		List<SetInfo> res = Lists.newArrayList();

		for (DBObject obj : sets) {

			res.add(this.getSetFromDBObject(obj));

		}

		return res;

	}

	private SetInfo getSetFromDBObject(final DBObject obj) {

		SetInfo setInfo = new SetInfo();

		setInfo.setEnabled((Boolean) obj.get("enabled"));

		setInfo.setQuery((String) obj.get("query"));

		setInfo.setSetDescription((String) obj.get("description"));

		setInfo.setSetName((String) obj.get("name"));

		setInfo.setSetSpec((String) obj.get("spec"));

		return setInfo;

	}

	private DBObject getObjectFromSet(final SetInfo s) {

		DBObject obj = BasicDBObjectBuilder.start("spec", s.getSetSpec()).add("name", s.getSetName()).add("description", s.getSetDescription())

				.add("query", s.getQuery()).add("enabled", s.isEnabled()).get();

		return obj;

	}

	private void ensureIndexesOnSets(final String dbName) {

		this.getSetsCollection(dbName).createIndex(new BasicDBObject("spec", 1), new IndexOptions().background(true));

		this.getSetsCollection(dbName).createIndex(new BasicDBObject("fromConfiguration", 1), new IndexOptions().background(true));

	}

	private void ensureIndexesOnCount(final String dbName) {

		BasicDBObject index = (BasicDBObject) BasicDBObjectBuilder.start("spec", 1).add("mdPrefix", 1).get();

		this.getSetsCountCollection(dbName).createIndex(index, new IndexOptions().background(true));

	}

	public MongoCollection<DBObject> getSetsCollection(final String dbName) {

		return this.getCollection(this.setCollection, dbName);

	}

	public MongoCollection<DBObject> getSetsCountCollection(final String dbName) {

		return this.getCollection(this.setCountCollection, dbName);

	}

	private MongoCollection<DBObject> getCollection(final String collectionName, final String dbName) {

		return publisherMongoClient.getDatabase(dbName).getCollection(collectionName, DBObject.class);

	}

	public String getSetCollection() {

		return setCollection;

	}

	public void setSetCollection(final String setCollection) {

		this.setCollection = setCollection;

	}

	public String getSetCountCollection() {

		return setCountCollection;

	}

	public void setSetCountCollection(final String setCountCollection) {

		this.setCountCollection = setCountCollection;

	}

	public MongoClient getPublisherMongoClient() {

		return publisherMongoClient;

	}

	public void setPublisherMongoClient(final MongoClient publisherMongoClient) {

		this.publisherMongoClient = publisherMongoClient;

	}

}

package eu.dnetlib.data.oai.store.mongo;

import java.util.Iterator;

import com.google.common.collect.Lists;

import com.mongodb.DBObject;

import com.mongodb.client.MongoCursor;

import eu.dnetlib.data.information.oai.publisher.info.RecordInfo;

import eu.dnetlib.data.oai.store.Cursor;

import eu.dnetlib.miscutils.functional.UnaryFunction;

public class DNetOAIMongoCursor implements Cursor {

	/**

	 * Underlying mongo cursor.

	 */

	private MongoCursor<DBObject> dbCursor;

	private int size = 0;

	/**

	 * Function to apply to records before delivering.

	 */

	private UnaryFunction<String, String> function;

	/**

	 * true if the RecordInfo returned by this Cursor must include the record body, false otherwise.

	 */

	private boolean bodyIncluded;

	private RecordInfoGenerator recordInfoGenerator;

	private MetadataExtractor metadataExtractor;

	private ProvenanceExtractor provenanceExtractor;

	public DNetOAIMongoCursor() {

		super();

	}

	public DNetOAIMongoCursor(final MongoCursor<DBObject> dbCursor, final boolean bodyIncluded, final RecordInfoGenerator recordInfoGenerator,

			final MetadataExtractor metadataExtractor) {

		this(dbCursor, null, bodyIncluded, recordInfoGenerator, metadataExtractor);

	}

	public DNetOAIMongoCursor(final MongoCursor<DBObject> dbCursor, final UnaryFunction<String, String> function, final boolean bodyIncluded,

			final RecordInfoGenerator recordInfoGenerator, final MetadataExtractor metadataExtractor) {

		super();

		this.dbCursor = dbCursor;

		this.size = 0;

		this.function = function;

		this.bodyIncluded = bodyIncluded;

		this.recordInfoGenerator = recordInfoGenerator;

		this.metadataExtractor = metadataExtractor;

	}

	/**

	 *

	 * {@inheritDoc}

	 */

	@Override

	public int count() {

		//I can do it because MongoCursor are always created from queries with "limit", so I do not expect the creation of the list to explode

		//to not exagerate, I'll get the size only if the current size is 0

		if (size == 0)

			size = Lists.newArrayList(dbCursor).size();

		return size;

	}

	/**

	 *

	 * {@inheritDoc}

	 *

	 * @see java.lang.Iterable#iterator()

	 */

	@Override

	public Iterator<RecordInfo> iterator() {

		return new Iterator<RecordInfo>() {

			@Override

			public boolean hasNext() {

				return dbCursor.hasNext();

			}

			@Override

			public RecordInfo next() {

				DBObject res = dbCursor.next();

				RecordInfo info = recordInfoGenerator.transformDBObject(res, bodyIncluded);

				if ((function != null) && bodyIncluded && (info != null)) {

					info.setMetadata(function.evaluate(info.getMetadata()));

				}

				return info;

			}

			@Override

			public void remove() {

				throw new UnsupportedOperationException();

			}

		};

	}

	public UnaryFunction<String, String> getFunction() {

		return function;

	}

	public void setFunction(final UnaryFunction<String, String> function) {

		this.function = function;

	}

	public MongoCursor<DBObject> getDbCursor() {

		return dbCursor;

	}

	public void setDbCursor(final MongoCursor<DBObject> dbCursor) {

		this.dbCursor = dbCursor;

	}

	@Override

	public boolean isBodyIncluded() {

		return this.bodyIncluded;

	}

	@Override

	public void setBodyIncluded(final boolean bodyIncluded) {

		this.bodyIncluded = bodyIncluded;

	}

	public RecordInfoGenerator getRecordInfoGenerator() {

		return recordInfoGenerator;

	}

	public void setRecordInfoGenerator(final RecordInfoGenerator recordInfoGenerator) {

		this.recordInfoGenerator = recordInfoGenerator;

	}

	public MetadataExtractor getMetadataExtractor() {

		return metadataExtractor;

	}

	public void setMetadataExtractor(final MetadataExtractor metadataExtractor) {

		this.metadataExtractor = metadataExtractor;

	}

	public ProvenanceExtractor getProvenanceExtractor() {

		return provenanceExtractor;

	}

	public void setProvenanceExtractor(final ProvenanceExtractor provenanceExtractor) {

		this.provenanceExtractor = provenanceExtractor;

	}

}

package eu.dnetlib.data.oai.store.mongo;

import java.util.List;

import javax.annotation.Resource;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.mongodb.DBObject;

import com.mongodb.MongoClient;

import com.mongodb.WriteConcern;

import com.mongodb.client.FindIterable;

import com.mongodb.client.MongoDatabase;

import com.mongodb.client.model.Filters;

import eu.dnetlib.data.information.oai.publisher.conf.OAIConfigurationReader;

import eu.dnetlib.data.oai.store.RecordChangeDetector;

import eu.dnetlib.data.oai.store.sets.MongoSetCollection;

import eu.dnetlib.functionality.cql.mongo.MongoCqlTranslator;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.cache.annotation.CacheEvict;

import org.springframework.cache.annotation.Cacheable;

/**

 * Created by alessia on 19/07/16.

 */

public class MongoPublisherCacheHelper {

	private static final Log log = LogFactory.getLog(MongoPublisherCacheHelper.class);

	@Autowired

	private MongoClient publisherMongoClient;

	@Resource(name = "oaiConfigurationExistReader")

	private OAIConfigurationReader configuration;

	@Resource

	private MetadataExtractor metadataExtractor;

	@Resource

	private RecordInfoGenerator recordInfoGenerator;

	@Resource

	private RecordChangeDetector recordChangeDetector;

	public  MongoDatabase getDB(final String dbName) {

		return this.publisherMongoClient.getDatabase(dbName).withWriteConcern(WriteConcern.JOURNALED);

	}

	@Cacheable(value="oaistores", key="#dbname")

	public List<MongoPublisherStore> listPublisherStores(final String dbName, final String metadataCollectionName, final boolean alwaysNewRecord, final MongoSetCollection mongoSetCollection) {

		log.info("Not using cache for listPublisherStores on "+dbName);

		final MongoDatabase db = getDB(dbName);

		final FindIterable<DBObject> stores = db.getCollection(metadataCollectionName, DBObject.class).find();

		return Lists.newArrayList(

				Iterables.transform(stores, new Function<DBObject, MongoPublisherStore>() {

					@Override

					public MongoPublisherStore apply(final DBObject storeInfo) {

						return createFromDBObject(storeInfo, db, alwaysNewRecord, mongoSetCollection);

					}

				})

		);

	}

	@Cacheable(value="oaistoresById", key="#storeId + #dbName", unless="#result == null")

	public MongoPublisherStore getStoreById(final String storeId, final String dbName, final String metadataCollectionName, final boolean alwaysNewRecord, final MongoSetCollection mongoSetCollection) {

		log.info(String.format("Not using cache for getStoreById: %s", storeId));

		DBObject storeInfo = getDB(dbName).getCollection(metadataCollectionName, DBObject.class).find(Filters.eq("id", storeId)).first();

		log.info("Got DBObject from mongo "+dbName+"."+metadataCollectionName+", id "+storeId+" is : "+storeInfo );

		return this.createFromDBObject(storeInfo, getDB(dbName), alwaysNewRecord, mongoSetCollection);

	}

	@CacheEvict(value="oaistoresById", key = "#storeId + #dbName")

	public void deleteFromCache(String storeId, String dbName){

		log.info("Evicting "+storeId+" for db "+dbName+ " from the cache");

	}

	private MongoPublisherStore createFromDBObject(final DBObject storeInfo, final MongoDatabase db, final boolean alwaysNewRecord, final MongoSetCollection mongoSetCollection) {

		if (storeInfo == null){

			log.error("cannot create MongoPublisherStore from null DBObject");

			return null;

		}

		log.debug("Creating MongoPublisherStore from DBObject "+storeInfo.toString());

		String storeId = (String) storeInfo.get("id");

		String mdFormat = (String) storeInfo.get("metadataFormat");

		String mdInterpretation = (String) storeInfo.get("interpretation");

		String mdLayout = (String) storeInfo.get("layout");

		MongoPublisherStore store = new MongoPublisherStore(storeId, mdFormat, mdInterpretation, mdLayout, db.getCollection(storeId, DBObject.class),

				this.configuration.getFields(mdFormat, mdInterpretation, mdLayout), recordInfoGenerator, this.configuration.getIdScheme(),

				this.configuration.getIdNamespace(), this.metadataExtractor, this.recordChangeDetector, alwaysNewRecord, db);

		store.setMongoSetCollection(mongoSetCollection);

		return store;

	}

}

package eu.dnetlib.data.oai.store.mongo;

import java.io.ByteArrayInputStream;

import java.io.IOException;

import java.io.StringReader;

import java.util.Date;

import java.util.List;

import java.util.zip.ZipEntry;

import java.util.zip.ZipInputStream;

import javax.annotation.Resource;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang.StringEscapeUtils;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.io.SAXReader;

import com.google.common.collect.Sets;

import com.mongodb.DBObject;

import eu.dnetlib.data.information.oai.publisher.OaiPublisherRuntimeException;

import eu.dnetlib.data.information.oai.publisher.conf.OAIConfigurationReader;

import eu.dnetlib.data.information.oai.publisher.info.RecordInfo;

/**

 * Helper class to generate a RecordInfo object from a Mongo DBObject.

 *

 * @author alessia

 *

 */

public class RecordInfoGenerator {

	@Resource

	private MetadataExtractor metadataExtractor;

	@Resource

	private ProvenanceExtractor provenanceExtractor;

	@SuppressWarnings("unchecked")

	public RecordInfo transformDBObject(final DBObject object, final boolean includeBody) {

		if ((object == null) || object.keySet().isEmpty()) return null;

		String id = (String) object.get(OAIConfigurationReader.ID_FIELD);

		// need to escape the identifier, otherwise the XML breaks

		id = StringEscapeUtils.escapeXml(id);

		boolean deleted = (Boolean) object.get("deleted");

		RecordInfo record = new RecordInfo();

		record.setIdentifier(id);

		record.setInternalId(object.get("_id").toString());

		record.setDatestamp((Date) object.get(OAIConfigurationReader.DATESTAMP_FIELD));

		record.setDeleted(deleted);

		List<String> sets = (List<String>) object.get(OAIConfigurationReader.SET_FIELD);

		if (sets != null) {

			record.setSetspecs(Sets.newHashSet(sets));

		}

		if (includeBody && !deleted) {

			String body = decompressRecord((byte[]) object.get(OAIConfigurationReader.BODY_FIELD));

			final SAXReader reader = new SAXReader();

			Document doc;

			try {

				doc = reader.read(new StringReader(body));

				record.setMetadata(this.metadataExtractor.evaluate(doc));

				record.setProvenance(this.provenanceExtractor.evaluate(doc));

			} catch (DocumentException e) {

				throw new OaiPublisherRuntimeException(e);

			}

		}

		return record;

	}

	public  String decompressRecord(final byte[] input)  {

		try {

			ByteArrayInputStream bis = new ByteArrayInputStream(input);

			ZipInputStream zis = new ZipInputStream(bis);

			ZipEntry ze;

			ze = zis.getNextEntry();

			if (ze== null)

				throw new OaiPublisherRuntimeException("cannot decompress null zip entry ");

			if (!ze.getName().equals(OAIConfigurationReader.BODY_FIELD))

				throw new OaiPublisherRuntimeException("cannot decompress zip entry name :"+ze.getName());

			return IOUtils.toString(zis);

		} catch (IOException e) {

			throw new OaiPublisherRuntimeException(e);

		}

	}

	public MetadataExtractor getMetadataExtractor() {

		return metadataExtractor;

	}

	public void setMetadataExtractor(final MetadataExtractor metadataExtractor) {

		this.metadataExtractor = metadataExtractor;

	}

	public ProvenanceExtractor getProvenanceExtractor() {

		return provenanceExtractor;

	}

	public void setProvenanceExtractor(final ProvenanceExtractor provenanceExtractor) {

		this.provenanceExtractor = provenanceExtractor;

	}

}

package eu.dnetlib.data.oai.store.mongo;

import java.io.IOException;

import java.util.Collection;

import java.util.Date;

import java.util.List;