D-Net

package eu.dnetlib.data.oai.store.sets;

import static org.junit.Assert.assertEquals;

import org.junit.Before;

import org.junit.Test;

public class MongoSetCollectionTest {

	private MongoSetCollection mongoSetCollection;

	private String strangeSet = "Наукові журнали Національного Авіаційного Університету";

	@Before

	public void setup() {

		this.mongoSetCollection = new MongoSetCollection();

	}

	@Test

	public void test() {

		String normalised = this.mongoSetCollection.normalizeSetSpec(strangeSet);

		assertEquals(MongoSetCollection.DEFAULT_SET, normalised);

	}

}

/**

 *

 */

package eu.dnetlib.data.oai.store.mongo;

import java.io.IOException;

import org.bson.types.Binary;

import org.junit.Assert;

import org.junit.Test;

/**

 * @author sandro

 *

 */

public class MongoPublisherStoreTest {

	/**

	 * Test method for {@link eu.dnetlib.data.oai.store.mongo.MongoPublisherStore#createCompressRecord(java.lang.String)}.

	 * @throws IOException

	 */

	@Test

	public void testCreateCompressRecord() throws IOException {

		MongoPublisherStore store = new MongoPublisherStore();

		RecordInfoGenerator gen = new RecordInfoGenerator();

		StringBuffer buffer = new StringBuffer();

		String  input =" CIAO MONDO DA SANDRO!";

		for (int i=0; i< 10000; i++) {

			buffer.append(input);

		}

		Binary data = store.createCompressRecord(buffer.toString());

		Assert.assertEquals(buffer.toString(),gen.decompressRecord(data.getData()));

	}

}

package eu.dnetlib.data.oai.store.mongo;

import java.io.StringReader;

import org.apache.commons.io.IOUtils;

import org.dom4j.Document;

import org.dom4j.io.SAXReader;

import org.junit.After;

import org.junit.Before;

import org.junit.Test;

import org.springframework.core.io.ClassPathResource;

/**

 * MetadataExtractor Tester.

 *

 * @author alessia

 * @version 1.0

 * @since <pre>Apr 6, 2016</pre>

 */

public class MetadataExtractorTest {

	final MetadataExtractor extractor = new MetadataExtractor();

	final SAXReader reader = new SAXReader();

	private Document doc;

	private String filePath = "/eu/dnetlib/data/oai/store/mongo/testRecord.xml";

	@Before

	public void before() throws Exception {

		String testRecord = IOUtils.toString(new ClassPathResource(filePath).getInputStream());

		doc = reader.read(new StringReader(testRecord));

	}

	@After

	public void after() throws Exception {

	}

	/**

	 * Method: evaluate(final Document xmlDoc)

	 */

	@Test

	public void testEvaluate() throws Exception {

		long timeStart = System.currentTimeMillis();

		String metadata = extractor.evaluate(doc);

		long timeEnd = System.currentTimeMillis();

		System.out.println("Got metadata in ms " + (timeEnd - timeStart));

		System.out.println(metadata);

	}

}

package eu.dnetlib.data.oai.store.mongo;

import java.io.StringReader;

import org.apache.commons.io.IOUtils;

import org.dom4j.Document;

import org.dom4j.io.SAXReader;

import org.junit.After;

import org.junit.Before;

import org.junit.Test;

import org.springframework.core.io.ClassPathResource;

/**

 * ProvenanceExtractor Tester.

 *

 * @author <Authors name>

 * @version 1.0

 * @since <pre>Apr 6, 2016</pre>

 */

public class ProvenanceExtractorTest {

	final ProvenanceExtractor extractor = new ProvenanceExtractor();

	final SAXReader reader = new SAXReader();

	private Document doc;

	private String filePath = "/eu/dnetlib/data/oai/store/mongo/testRecord.xml";

	@Before

	public void before() throws Exception {

		String testRecord = IOUtils.toString(new ClassPathResource(filePath).getInputStream());

		doc = reader.read(new StringReader(testRecord));

	}

	@After

	public void after() throws Exception {

	}

	/**

	 * Method: evaluate(final Document xmlDoc)

	 */

	@Test

	public void testEvaluate() throws Exception {

		long timeStart = System.currentTimeMillis();

		String prov = extractor.evaluate(doc);

		long timeEnd = System.currentTimeMillis();

		System.out.println("Got provenance in ms " + (timeEnd - timeStart));

		System.out.println(prov);

	}

}

package eu.dnetlib.data.oai.store.mongo;

import java.io.IOException;

import java.util.zip.ZipEntry;

import java.util.zip.ZipOutputStream;

import eu.dnetlib.data.information.oai.publisher.conf.OAIConfigurationReader;

import org.apache.commons.io.IOUtils;

import org.apache.commons.io.output.ByteArrayOutputStream;

import org.bson.types.Binary;

import org.junit.Before;

import org.junit.Test;

import org.springframework.core.io.ClassPathResource;

/**

 * RecordInfoGenerator Tester.

 *

 * @author <Authors name>

 * @version 1.0

 * @since <pre>Apr 6, 2016</pre>

 */

public class RecordInfoGeneratorTest {

	private RecordInfoGenerator gen = new RecordInfoGenerator();

	private String filePath = "/eu/dnetlib/data/oai/store/mongo/testRecord.xml";

	private Binary binaryXML;

	@Before

	public void before() throws Exception {

		String testRecord = IOUtils.toString(new ClassPathResource(filePath).getInputStream());

		binaryXML = createCompressRecord(testRecord);

	}

	/**

	 * Method: decompressRecord(final byte[] input)

	 */

	@Test

	public void testDecompressRecord() throws Exception {

		long timeStart = System.currentTimeMillis();

		String record = gen.decompressRecord(binaryXML.getData());

		long timeEnd = System.currentTimeMillis();

		System.out.println("Decompressed record in ms " + (timeEnd - timeStart));

		System.out.println(record);

	}

	private Binary createCompressRecord(final String record) throws IOException {

		ByteArrayOutputStream os = new ByteArrayOutputStream();

		ZipOutputStream zos = new ZipOutputStream(os);

		ZipEntry entry = new ZipEntry(OAIConfigurationReader.BODY_FIELD);

		zos.putNextEntry(entry);

		zos.write(record.getBytes());

		zos.closeEntry();

		//zos.flush();

		zos.close();

		return new Binary(os.toByteArray());

	}

}

<RESOURCE_PROFILE>

	<HEADER>

		<RESOURCE_IDENTIFIER value="OAIPublisherConfiguration" />

		<RESOURCE_TYPE value="OAIPublisherConfigurationDSResourceType" />

		<RESOURCE_KIND value="OAIPublisherConfigurationDSResources" />

		<RESOURCE_URI value="" />

		<DATE_OF_CREATION value="2001-12-31T12:00:00" />

	</HEADER>

	<BODY>

		    <CONFIGURATION>

            <OAISETS>

                <OAISET enabled="true">

                    <spec>OpenAccess</spec>

                    <name>Set of Open Access articles</name>

                    <description>Set of records having 'OPEN' license</description>

                    <query>(license = "OPEN")</query>

                </OAISET>

                <OAISET enabled="true">

                    <spec>ArticlesInNature</spec>

                    <name>Articles published by Nature</name>

                    <description>Set of articles published by Nature Publishing Group</description>

                    <query>(publisher = "Nature Publishing Group")</query>

                </OAISET>

                <OAISET enabled="true">

                    <spec>publications</spec>

                    <name>Publications</name>

                    <description>Set of all Publications</description>

                    <query>resulttypeid="publication"</query>

                </OAISET>

            </OAISETS>

            <METADATAFORMATS>

                <METADATAFORMAT exportable="true" metadataPrefix="oaf">

                    <NAMESPACE>http://namespace.openaire.eu/oaf</NAMESPACE>

                    <SCHEMA>http://www.openaire.eu/schema/0.1/oaf-0.1.xsd</SCHEMA>

                    <SOURCE_METADATA_FORMAT interpretation="openaire" layout="index" name="oaf"/>

                    <TRANSFORMATION_RULE/>

                    <BASE_QUERY>*</BASE_QUERY>

                </METADATAFORMAT>

                <METADATAFORMAT metadataPrefix="oai_dc" exportable="false">

                    <NAMESPACE>http://www.openarchives.org/OAI/2.0/oai_dc/</NAMESPACE>

                    <SCHEMA>http://www.openarchives.org/OAI/2.0/oai_dc.xsd</SCHEMA>

                    <SOURCE_METADATA_FORMAT interpretation="openaire" layout="index" name="oaf"/>

                    <TRANSFORMATION_RULE>oaf2dc_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=</TRANSFORMATION_RULE>

                    <BASE_QUERY>oaftype="result"</BASE_QUERY>

                </METADATAFORMAT>

            </METADATAFORMATS>

            <INDICES>

                <INDEX name="objIdentifier" repeatable="false">

                    <SOURCE name="oaf" layout="index" interpretation="openaire" path="//*[local-name() ='objIdentifier']"/>

                </INDEX>

                <INDEX name="set" repeatable="true">

                    <SOURCE name="oaf" layout="index" interpretation="openaire" path="//collectedfrom/@name"/>

                </INDEX>

                <INDEX name="publisher" repeatable="true">

                    <SOURCE name="oaf" layout="index" interpretation="openaire" path="//publisher"/>

                </INDEX>

                <INDEX name="license" repeatable="false">

                    <SOURCE name="oaf" layout="index" interpretation="openaire" path="//bestlicense/@classid"/>

                </INDEX>

                <INDEX name="oaftype" repeatable="false">

                    <SOURCE name="oaf" layout="index" interpretation="openaire" path="local-name(//*[local-name()='entity']/*)"/>

                </INDEX>

                <INDEX name="resulttypeid" repeatable="false">

                    <SOURCE name="oaf" layout="index" interpretation="openaire" path="//*[local-name()='entity']/*[local-name()='result']/resulttype/@classid"/>

                </INDEX>

            </INDICES>

        </CONFIGURATION>

		<STATUS>

			<LAST_UPDATE value="2001-12-31T12:00:00" />

		</STATUS>

		<SECURITY_PARAMETERS>SECURITY_PARAMETERS</SECURITY_PARAMETERS>

	</BODY>

</RESOURCE_PROFILE>

<?xml version="1.0" encoding="UTF-8"?>

<record xmlns:dc="http://purl.org/dc/elements/1.1/"

        xmlns:dr="http://www.driver-repository.eu/namespace/dr"

        xmlns:dri="http://www.driver-repository.eu/namespace/dri"

        xmlns:oaf="http://namespace.openaire.eu/oaf"

        xmlns:oai="http://www.openarchives.org/OAI/2.0/"

        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">

	<header xmlns="http://namespace.openaire.eu/">

		<dri:objIdentifier>od_______263::06235d139088d4b2ca45ace37c458a01</dri:objIdentifier>

		<dri:recordIdentifier>oai:DiVA.org:su-91808</dri:recordIdentifier>

		<dri:dateOfCollection/>

		<dri:mdFormat/>

		<dri:mdFormatInterpretation/>

		<dri:repositoryId>866c86e1-0e6a-4e12-b5ed-b7e0eb956381_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=</dri:repositoryId>

		<dr:objectIdentifier/>

		<dr:dateOfCollection>2016-04-05T06:52:25.98Z</dr:dateOfCollection>

		<dr:dateOfTransformation>2016-04-05T06:53:01.394Z</dr:dateOfTransformation>

		<oaf:datasourceprefix>od_______263</oaf:datasourceprefix>

	</header>

	<metadata xmlns="http://namespace.openaire.eu/">

		<oaf:entity>

			<dc:title>Evaluation and Tuning of Model Trajectoriesand Spreading Rates in the Baltic Sea Using Surface Drifter Observations</dc:title>

			<dc:creator>Kjellsson, Joakim</dc:creator>

			<dc:creator>Döös, Kristofer</dc:creator>

			<dc:creator>Soomere, Tarmo</dc:creator>

			<dc:date>2013</dc:date>

			<dc:description>Results from experiments with surface drifters in the Baltic Sea in 2010–2011 are presented and discussed. In a first experiment, 12 SVP-B (Surface Velocity Program, with Barometer) drifters with a drogue at 12–18 m depth were deployed in the Baltic Sea. In a second experiment, shallow drifters extending to a depth of 1.5 m were deployed in the Gulf of Finland. Results from the SVP-B drifter experiment are compared to results from a regional ocean model and a trajectory code. Differences between the observed SVP-B drifters and simulated drifters are found for absolute dispersion (i.e., squared displacement from initial position) and relative dispersion (i.e., squared distance between two initially paired drifters). The former is somewhat underestimated since the simulated currents are neither as fast nor as variable as those observed. The latter is underestimated both due to the above-mentioned reasons and due to the resolution of the ocean model. For the shallower drifters, spreading in the upper 1–2 m of the Gulf of Finland is investigated. The spreading rate is about 200 m/day for separations &lt;0.5 km, 500 m/day for separations below 1 km and in the range of 0.5–3 km/day for separations in the range of 1–4 km. The spreading rate does not follow Richardson’s law. The initial spreading, up to a distance of about d=100–150 m, is governed by the power law d∼t 0.27 whereas for larger separations the distance increases as d∼t2.5.</dc:description>

			<dc:description>BalticWay</dc:description>

			<dc:format>application/pdf</dc:format>

			<dc:identifier>http://urn.kb.se/resolve?urn=urn:nbn:se:su:diva-91808</dc:identifier>

			<dc:language>eng</dc:language>

			<dc:publisher>Stockholms universitet, Meteorologiska institutionen (MISU)</dc:publisher>

			<dc:publisher>Stockholms universitet, Meteorologiska institutionen (MISU)</dc:publisher>

			<dc:publisher>Institute of Cybernetics, Tallinn University of Technology</dc:publisher>

			<dc:relation>Preventive Methods for Coastal Protection : Towards the Use of Ocean Dynamics for Pollution Control, p. 251-281</dc:relation>

			<dc:relation>info:eu-repo/grantAgreement/EC/FP7/217246</dc:relation>

			<dc:subject>surface drifters; rco; ocean model; trajectory;</dc:subject>

			<dc:type>Chapter in book</dc:type>

			<dc:type>info:eu-repo/semantics/bookPart</dc:type>

			<dc:type>text</dc:type>

			<dr:CobjCategory>0013</dr:CobjCategory>

			<dr:CobjIdentifier>urn:isbn:978-3-319-00440-2</dr:CobjIdentifier>

			<dr:CobjIdentifier>urn:isbn:978-3-319-00439-6</dr:CobjIdentifier>

			<dr:CobjIdentifier>doi:10.1007/978-3-319-00440-2_8</dr:CobjIdentifier>

			<oaf:dateAccepted>2013-01-01</oaf:dateAccepted>

			<oaf:projectid>corda_______::217246</oaf:projectid>

			<oaf:collectedDatasourceid>opendoar____::263</oaf:collectedDatasourceid>

			<oaf:accessrights>OPEN</oaf:accessrights>

			<oaf:hostedBy id="opendoar____::263" name="Publikationer från Stockholms universitet"/>

			<oaf:collectedFrom id="opendoar____::263" name="Publikationer från Stockholms universitet"/>

			<oaf:identifier identifierType="doi">10.1007/978-3-319-00440-2_8</oaf:identifier>

		</oaf:entity>

	</metadata>

	<about xmlns="http://namespace.openaire.eu/">

		<provenance xmlns="http://www.openarchives.org/OAI/2.0/provenance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/provenance http://www.openarchives.org/OAI/2.0/provenance.xsd">

			<originDescription altered="true" harvestDate="2016-04-05T06:52:25.98Z">

				<baseURL>http://su.diva-portal.org/dice/oai</baseURL>

				<identifier>oai:DiVA.org:su-91808</identifier>

				<datestamp>2013-07-05T10:47:00Z</datestamp>

				<metadataNamespace>http://www.openarchives.org/OAI/2.0/oai_dc/</metadataNamespace>

			</originDescription>

		</provenance>

		<oaf:datainfo>

			<oaf:inferred>false</oaf:inferred>

			<oaf:deletedbyinference>false</oaf:deletedbyinference>

			<oaf:trust>0.9</oaf:trust>

			<oaf:inferenceprovenance/>

			<oaf:provenanceaction classid="sysimport:crosswalk:repository"

			                      classname="sysimport:crosswalk:repository"

			                      schemeid="dnet:provenanceActions" schemename="dnet:provenanceActions"/>

		</oaf:datainfo>

	</about>

</record>

org.apache.cxf.Logger=org.apache.cxf.common.logging.Log4jLogger

log4j.rootLogger=WARN, CONSOLE

log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender

log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout

log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n

log4j.logger.eu.dnetlib=INFO

log4j.logger.eu.dnetlib.data=DEBUG

<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.0">

    <xsl:template match="//*[local-name()='metadata']">

        <metadata>

            <dmf>

                <xsl:apply-templates/>

            </dmf>

        </metadata>

    </xsl:template>

    <xsl:template match="@*|node()">

        <xsl:copy>

            <xsl:apply-templates select="@*|node()"/>

        </xsl:copy>

    </xsl:template>

</xsl:stylesheet>

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

       xmlns:p="http://www.springframework.org/schema/p" xmlns="http://www.springframework.org/schema/beans"

       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-4.0.xsd">

	<bean id="mongodbPublisherStoreDao" class="eu.dnetlib.data.oai.store.mongo.MongoPublisherStoreDAO"

		p:metadataCollection="metadata" />

	<bean id="publisherMongoClient" class="com.mongodb.MongoClient">

		<constructor-arg index="0" type="com.mongodb.ServerAddress">

			<bean class="com.mongodb.ServerAddress">

				<constructor-arg index="0"

					value="${services.publisher.oai.host}" />

				<constructor-arg index="1"

					value="${services.publisher.oai.port}" />

			</bean>

		</constructor-arg>

	</bean>

	<bean id="recordChangeDetector" class="eu.dnetlib.data.oai.store.DummyRecordChangeDetector" />

	<bean id="metadataExtractor" class="eu.dnetlib.data.oai.store.mongo.MetadataExtractor" />

	<bean id="recordInfoGenerator" class="eu.dnetlib.data.oai.store.mongo.RecordInfoGenerator" />

	<bean id="mongoSetCollection" class="eu.dnetlib.data.oai.store.sets.MongoSetCollection" />

	<bean id="provenanceExtractor" class="eu.dnetlib.data.oai.store.mongo.ProvenanceExtractor" />

	<bean id="theOAICacheManager" class="org.springframework.cache.ehcache.EhCacheCacheManager"

        p:cache-manager-ref="oaiStoreCacheManager" />

	<bean id="oaiStoreCacheManager"

		class="org.springframework.cache.ehcache.EhCacheManagerFactoryBean"

		p:cacheManagerName="oaiStoreCacheManger"

		p:configLocation="classpath:/eu/dnetlib/data/oai/store/mongo/cache/ehcache.xml" />

</beans>

<ehcache>

	<!-- Sets the path to the directory where cache .data files are created. 

		If the path is a Java System Property it is replaced by its value in the 

		running VM. The following properties are translated: user.home - User's home 

		directory user.dir - User's current working directory java.io.tmpdir - Default 

		temp file path -->

	<!--<diskStore path="java.io.tmpdir" />-->

	<!--Default Cache configuration. These will applied to caches programmatically 

		created through the CacheManager. The following attributes are required for 

		defaultCache: maxInMemory - Sets the maximum number of objects that will 

		be created in memory eternal - Sets whether elements are eternal. If eternal, 

		timeouts are ignored and the element is never expired. timeToIdleSeconds 

		- Sets the time to idle for an element before it expires. i.e. The maximum 

		amount of time between accesses before an element expires Is only used if 

		the element is not eternal. Optional attribute. A value of 0 means that an 

		Element can idle for infinity timeToLiveSeconds - Sets the time to live for 

		an element before it expires. i.e. The maximum time between creation time 

		and when an element expires. Is only used if the element is not eternal. 

		overflowToDisk - Sets whether elements can overflow to disk when the in-memory 

		cache has reached the maxInMemory limit. -->

	<!-- <defaultCache maxElementsInMemory="1000" overflowToDisk="true" -->

	<!-- eternal="true" diskPersistent="false" timeToIdleSeconds="0" -->

	<!-- timeToLiveSeconds="0" memoryStoreEvictionPolicy="LRU" -->

	<!-- diskExpiryThreadIntervalSeconds="120" diskSpoolBufferSizeMB="5" /> -->

	<cache name="oaistores" maxElementsInMemory="100" eternal="false"

		timeToIdleSeconds="1800" timeToLiveSeconds="1800" overflowToDisk="false" />

	<cache name="oaistoresById" maxElementsInMemory="100"

		eternal="false" timeToIdleSeconds="1800" timeToLiveSeconds="1800"

		overflowToDisk="false" />

</ehcache>

services.publisher.oai.host=localhost

services.publisher.oai.port=27017

#services.publisher.oai.db=oaistore

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws"

	xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing"

	xmlns:p="http://www.springframework.org/schema/p" xmlns:http="http://cxf.apache.org/transports/http/configuration"

	xmlns:t="http://dnetlib.eu/springbeans/t" xmlns:template="http://dnetlib.eu/springbeans/template"

	xmlns:util="http://www.springframework.org/schema/util"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd

    	http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd

        http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd

        http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd

        http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd

        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd

        http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">

	<!-- endpoints -->

	<jaxws:endpoint id="oaiStoreServiceEndpoint"

		implementor="#oaiStoreService" implementorClass="eu.dnetlib.data.oai.store.OAIStoreService"

		address="/oaistore" />

	<template:instance name="serviceRegistrationManager"

		t:serviceRegistrationManagerClass="eu.dnetlib.enabling.tools.registration.ValidatingServiceRegistrationManagerImpl"

		t:name="oaiStoreServiceRegistrationManager" t:service="oaiStoreService"

		t:endpoint="oaiStoreServiceEndpoint" t:jobScheduler="jobScheduler"

		t:serviceRegistrator="blackboardServiceRegistrator" />

	<bean id="oaiStoreService" class="eu.dnetlib.data.oai.store.OAIStoreServiceImpl"

		init-method="start" destroy-method="stop" p:notificationHandler-ref="oaiStoreNotificationHandler" />

	<bean id="oaiStoreNotificationHandler"

		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerExecutorNotificationHandler"

		p:blackboardExecutor-ref="oaiStoreBlackboardExecutor" />

	<bean id="oaiStoreBlackboardExecutor"

		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerActionExecutor"

		p:blackboardHandler-ref="blackboardHandler"

		p:actionType="eu.dnetlib.data.oai.store.actions.OAIStoreActions"

		p:incomplete="false">

		<property name="actionMap">

			<map>

				<entry key="SYNC">

					<bean class="eu.dnetlib.data.oai.store.actions.SyncAction" />

				</entry>

				<entry key="COUNT_SETS">

					<bean class="eu.dnetlib.data.oai.store.actions.CountSetsAction"/>

				</entry>

				<entry key="REFRESH_CONFIG">

					<bean class="eu.dnetlib.data.oai.store.actions.RefreshConfigAction" />

				</entry>

				<entry key="ENSURE_INDEXES">

					<bean class="eu.dnetlib.data.oai.store.actions.EnsureIndexesAction" />

				</entry>

				<entry key="CREATE_STORE">

					<bean class="eu.dnetlib.data.oai.store.actions.CreateStoreAction" />

				</entry>

				<entry key="CREATE_OAI_INDEX">

					<bean class="eu.dnetlib.data.oai.store.actions.CreateOAIIndexAction" />

				</entry>

				<entry key="DROP_STORE">

					<bean class="eu.dnetlib.data.oai.store.actions.DropStoreAction" />

				</entry>

			</map>

		</property>

	</bean>

	<bean id="synchronizer" class="eu.dnetlib.data.oai.store.sync.OAIStoreSynchronizer" />

	<bean id="oaiHelper" class="eu.dnetlib.data.oai.store.conf.OAIHelper"/>

	<bean id="oaiSetsCounter" class="eu.dnetlib.data.oai.store.conf.OAISetsCounter" />

	<bean id="mongoCacheHelper" class="eu.dnetlib.data.oai.store.mongo.MongoPublisherCacheHelper"/>

</beans>

package eu.dnetlib.data.oai.store;

import eu.dnetlib.data.information.oai.publisher.info.RecordInfo;

public interface Cursor extends Iterable<RecordInfo> {

	int count();

	boolean isBodyIncluded();

	void setBodyIncluded(boolean bodyIncluded);

}

package eu.dnetlib.data.oai.store;

import java.util.List;

import eu.dnetlib.data.information.oai.publisher.OaiPublisherException;

public interface PublisherStoreDAO<X extends PublisherStore<T>, T extends Cursor> {

	/**

	 * Lists all PublisherStore.

	 * 

	 * @param dbName

	 *            name of the target database

	 * 

	 * @return a List of PublisherStore instances.

	 */

	List<X> listPublisherStores(final String dbName);

	/**

	 * Gets the store with the given identifier.

	 * 

	 * @param dbName

	 *            name of the target database

	 * 

	 * @param storeId

	 *            identifier of the store to retrieve

	 * @return a PublisherStore instance or null if there is no store with the given id.

	 */

	X getStore(final String storeId, final String dbName);

	/**

	 * Gets the store with the given properties.

	 * 

	 * @param mdfName

	 *            name of the metadata format

	 * @param mdfInterpretation

	 *            name of the metadata interpretation

	 * @param mdfLayout

	 *            name of the metadata layout

	 * @param dbName

	 *            name of the target database

	 * @return a PublisherStore instance or null if there is no store with the given properties.

	 */

	X getStore(final String mdfName, final String mdfInterpretation, final String mdfLayout, final String dbName);

	/**

	 * Gets the store to be used as source to deliver records with the given metadata prefix.

	 * 

	 * @param targetMetadataPrefix

	 *            prefix of the metadata format deliverable through this store

	 * @param dbName

	 *            name of the target database

	 * @return a PublisherStore instance or null if there is no store serving the given metadata prefix.

	 */

	X getStoreFor(final String targetMetadataPrefix, final String dbName);

	/**

	 * Create a PublisherStore with the given properties.

	 * 

	 * @param mdfName

	 *            name of the metadata format

	 * @param mdfInterpretation

	 *            name of the metadata interpretation

	 * @param mdfLayout

	 *            name of the metadata layout

	 * @param dbName

	 *            name of the target database

	 * @return a PublisherStore instance whose identifier is automatically generated.

	 * @throws OaiPublisherException

	 *             if there is already another PublisherStore with the given metadata format name, layout and interpretation.

	 */

	X createStore(final String mdfName, final String mdfInterpretation, final String mdfLayout, final String dbName) throws OaiPublisherException;

	/**

	 * Deletes the store with the given identifier.

	 * 

	 * @param storeId

	 *            id of the store to delete

	 * @param dbName

	 *            name of the target database

	 * @return true if the store was deleted successfully, false otherwise (e.g., a store with the given id does not exist).

	 */

	boolean deleteStore(final String storeId, final String dbName);

	/**

	 * Deletes from the store with the given identifier the records belonging to the given set.

	 * 

	 * @param storeId

	 *            id of the store to delete

	 * @param dbName

	 *            name of the target database

	 * @param set

	 *            name of the set

	 * @return true if the records were deleted successfully, false otherwise (e.g., a store with the given id does not exist).

	 */

	boolean deleteFromStore(final String storeId, final String dbName, final String set);

	/**

	 * Deletes from the store with the given identifier the records belonging to the given set.

	 * 

	 * @param mdfName

	 *            name of the metadata format

	 * @param mdfInterpretation

	 *            name of the metadata interpretation

	 * @param mdfLayout

	 *            name of the metadata layout

	 * @param dbName

	 *            name of the target database

	 * @param set

	 *            name of the set

	 * @return true if the records were deleted successfully, false otherwise (e.g., a store with the given id does not exist).

	 */

	boolean deleteFromStore(final String mdfName, final String mdfInterpretation, final String mdfLayout, final String dbName, final String set);

	/**

	 * Deletes the store with the given properties.

	 * 

	 * @param mdfName

	 *            name of the metadata format

	 * @param mdfInterpretation

	 *            name of the metadata interpretation

	 * @param mdfLayout

	 *            name of the metadata layout

	 * @param dbName

	 *            name of the target database

	 * @return true if the store was deleted successfully, false otherwise (e.g., a store with the given properties does not exist).

	 */

	boolean deleteStore(final String mdfName, final String mdfInterpretation, final String mdfLayout, final String dbName);

}

package eu.dnetlib.data.oai.store;

public interface RecordChangeDetector {

	/**

	 * Checks if the two records have differences based on logics that vary on the actual implementor class.

	 * 

	 * @param record1

	 *            first record to compare

	 * @param record2

	 *            second record to compare

	 * @return true if the two records differ based on the implementor's logics

	 */

	boolean differs(final String record1, final String record2);

}

package eu.dnetlib.data.oai.store.sets;

import java.text.Normalizer;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.mongodb.BasicDBObject;

import com.mongodb.BasicDBObjectBuilder;

import com.mongodb.DBObject;

import com.mongodb.MongoClient;

import com.mongodb.client.FindIterable;

import com.mongodb.client.MongoCollection;

import com.mongodb.client.model.Filters;

import com.mongodb.client.model.FindOneAndReplaceOptions;

import com.mongodb.client.model.FindOneAndUpdateOptions;

import com.mongodb.client.model.IndexOptions;

import eu.dnetlib.data.information.oai.publisher.info.SetInfo;

import eu.dnetlib.data.information.oai.sets.SetCollection;

import org.apache.commons.lang.StringEscapeUtils;

import org.apache.commons.lang.StringUtils;

import org.bson.conversions.Bson;

import org.springframework.beans.factory.annotation.Autowired;

public class MongoSetCollection implements SetCollection {

	public static String DEFAULT_SET = "OTHER";

	@Autowired

	private MongoClient publisherMongoClient;

	private String setCollection = "sets";

	private String setCountCollection = "setsCount";

	public void ensureIndexes(final String dbName) {

		this.ensureIndexesOnSets(dbName);

		this.ensureIndexesOnCount(dbName);

	}

	@Override

	public List<SetInfo> getAllSets(final boolean enabledOnly, final String dbName) {

		FindIterable<DBObject> iter = null;

		if (!enabledOnly) {

			iter = this.getSetsCollection(dbName).find();

		} else {

			Bson where = Filters.eq("enabled", true);

			iter = this.getSetsCollection(dbName).find(where);

		}

		return Lists.newArrayList(Iterables.transform(iter, new Function<DBObject, SetInfo>() {

			@Override

			public SetInfo apply(final DBObject dbObject) {

				return getSetFromDBObject(dbObject);

			}

		}));

	}

	@Override

	public boolean containSet(final String set, final String dbName) {

		Bson query = Filters.eq("spec", set);

		return this.getSetsCollection(dbName).count(query) != 0;

	}

	@Override

	public boolean containEnabledSet(final String set, final String publisherDBName) {

		Bson query = Filters.and(Filters.eq("spec", set), Filters.eq("enabled", true));

		return this.getSetsCollection(publisherDBName).count(query) != 0;

	}

	@Override

	public String getSetQuery(final String set, final String dbName) {

		Bson query = Filters.eq("spec", set);

		BasicDBObject returnField = new BasicDBObject("query", 1);

		DBObject obj = this.getSetsCollection(dbName).find(query).projection(returnField).first();

		return (String) obj.get("query");

	}

	@Override

	public int count(final String setSpec, final String mdPrefix, final String dbName) {

		Bson query = Filters.and(Filters.eq("spec", setSpec), Filters.eq("mdPrefix", mdPrefix));

		BasicDBObject returnField = new BasicDBObject("count", 1);

		DBObject obj = this.getSetsCountCollection(dbName).find(query).projection(returnField).first();

		if (obj == null) return 0;

		return (Integer) obj.get("count");

	}

	public void updateCounts(final String setSpec, final String mdPrefix, final int count, final String dbName) {

		BasicDBObject countUpdate = new BasicDBObject("$set", new BasicDBObject("count", count));

		Bson query = Filters.and(Filters.eq("spec", setSpec), Filters.eq("mdPrefix", mdPrefix));

		this.getSetsCountCollection(dbName).findOneAndUpdate(query, countUpdate, new FindOneAndUpdateOptions().upsert(true));

	}

	public void upsertSet(final SetInfo setInfo, final boolean fromConfiguration, final String dbName) {

		DBObject obj = this.getObjectFromSet(setInfo);

		obj.put("fromConfiguration", fromConfiguration);

		//this.getSetsCollection(dbName).update(new BasicDBObject("spec", setInfo.getSetSpec()), obj, true, false);

		this.getSetsCollection(dbName).findOneAndReplace(Filters.eq("spec", setInfo.getSetSpec()), obj, new FindOneAndReplaceOptions().upsert(true));

	}

	public String normalizeSetSpec(final String setName) {

		String s = StringEscapeUtils.unescapeXml(setName);

		s = Normalizer.normalize(s, Normalizer.Form.NFD);

		// replace spaces with underscores

		s = s.replaceAll(" ", "_");

		// remove tilde, dots... over letters

		s = s.replaceAll("[\\p{InCombiningDiacriticalMarks}&&[^-_]]", "");

		// change punctuation into an underscore

		s = s.replaceAll("[\\p{Punct}&&[^-_]]", "_");

		// remove all non-word characters

		s = s.replaceAll("[\\W&&[^-_]]", "");

		// Avoiding set '___' generated when we have "strange" set names such as those in cyrillic/ukrain

		// strips _ from the beginning and the end

		String stripped = StringUtils.strip(s, "_ ");

		if (StringUtils.isBlank(stripped)) {

			stripped = DEFAULT_SET;

		}

		return stripped;

	}

	public List<SetInfo> getConfiguredSets(final String dbName) {

		Bson query = Filters.eq("fromConfiguration", true);

		return this.findSets(query, dbName);

	}

	public List<SetInfo> getSetsFromData(final String dbName) {

		Bson query = Filters.eq("fromConfiguration", false);

		return this.findSets(query, dbName);

	}

	public void dropOAISets(final String dbName) {

		this.getSetsCountCollection(dbName).drop();

		this.getSetsCollection(dbName).drop();

	}

	public void dropSet(final String dbName, final String setSpec) {

		Bson query = Filters.eq("spec", setSpec);

		this.getSetsCollection(dbName).deleteMany(query);

		this.getSetsCountCollection(dbName).deleteMany(query);

	}

	public void dropConfigurationSets(final String dbName) {

		this.getSetsCollection(dbName).deleteMany(Filters.eq("fromConfiguration", true));

	}

	protected List<SetInfo> findSets(final Bson query, final String dbName) {

		final FindIterable<DBObject> sets = this.getSetsCollection(dbName).find(query);

		List<SetInfo> res = Lists.newArrayList();

		for (DBObject obj : sets) {

			res.add(this.getSetFromDBObject(obj));

		}

		return res;

	}

	private SetInfo getSetFromDBObject(final DBObject obj) {

		SetInfo setInfo = new SetInfo();

		setInfo.setEnabled((Boolean) obj.get("enabled"));

		setInfo.setQuery((String) obj.get("query"));

		setInfo.setSetDescription((String) obj.get("description"));

		setInfo.setSetName((String) obj.get("name"));

		setInfo.setSetSpec((String) obj.get("spec"));

		return setInfo;

	}

	private DBObject getObjectFromSet(final SetInfo s) {

		DBObject obj = BasicDBObjectBuilder.start("spec", s.getSetSpec()).add("name", s.getSetName()).add("description", s.getSetDescription())

				.add("query", s.getQuery()).add("enabled", s.isEnabled()).get();

		return obj;

	}

	private void ensureIndexesOnSets(final String dbName) {

		this.getSetsCollection(dbName).createIndex(new BasicDBObject("spec", 1), new IndexOptions().background(true));

		this.getSetsCollection(dbName).createIndex(new BasicDBObject("fromConfiguration", 1), new IndexOptions().background(true));

	}

	private void ensureIndexesOnCount(final String dbName) {

		BasicDBObject index = (BasicDBObject) BasicDBObjectBuilder.start("spec", 1).add("mdPrefix", 1).get();

		this.getSetsCountCollection(dbName).createIndex(index, new IndexOptions().background(true));

	}

	public MongoCollection<DBObject> getSetsCollection(final String dbName) {

		return this.getCollection(this.setCollection, dbName);

	}

	public MongoCollection<DBObject> getSetsCountCollection(final String dbName) {

		return this.getCollection(this.setCountCollection, dbName);

	}

	private MongoCollection<DBObject> getCollection(final String collectionName, final String dbName) {

		return publisherMongoClient.getDatabase(dbName).getCollection(collectionName, DBObject.class);

	}

	public String getSetCollection() {

		return setCollection;

	}

	public void setSetCollection(final String setCollection) {

		this.setCollection = setCollection;

	}

	public String getSetCountCollection() {

		return setCountCollection;

	}

	public void setSetCountCollection(final String setCountCollection) {

		this.setCountCollection = setCountCollection;

	}

	public MongoClient getPublisherMongoClient() {

		return publisherMongoClient;

	}

	public void setPublisherMongoClient(final MongoClient publisherMongoClient) {

		this.publisherMongoClient = publisherMongoClient;

	}

}

package eu.dnetlib.data.oai.store.mongo;

import eu.dnetlib.miscutils.functional.UnaryFunction;

import org.dom4j.Document;

import org.dom4j.Node;

/**

 * Function to skip the header of the record and deliver only its metadata content as XML String.

 */

public class MetadataExtractor implements UnaryFunction<String, Document> {

	@Override

	public String evaluate(final Document xmlDoc) {

		Node metadataNode = xmlDoc.selectSingleNode("//*[local-name() = 'metadata']/*");

		//Node metadataNode = xmlDoc.selectSingleNode("/*[local-name()='record']/*[local-name() = 'metadata']/*");

		return metadataNode.asXML();

	}

}

package eu.dnetlib.data.oai.store.mongo;

import eu.dnetlib.miscutils.functional.UnaryFunction;

import org.dom4j.Document;

import org.dom4j.Node;

/**

 * Function to deliver only the about/provenance content of a record as XML String.

 */

public class ProvenanceExtractor implements UnaryFunction<String, Document> {

	@Override

	public String evaluate(final Document xmlDoc) {

		//	Node provenanceNode = xmlDoc.selectSingleNode("//*[local-name() = 'about']/*[local-name() = 'provenance']");

		Node provenanceNode = xmlDoc.selectSingleNode("/*[local-name()='record']/*[local-name() = 'about']/*[local-name() = 'provenance']");

		if (provenanceNode != null) return provenanceNode.asXML();

		else return null;

	}

}

package eu.dnetlib.data.oai.store.mongo;

import java.util.List;

import javax.annotation.Resource;

import com.mongodb.BasicDBObjectBuilder;

import com.mongodb.DBObject;

import com.mongodb.client.MongoCollection;

import com.mongodb.client.MongoDatabase;

import com.mongodb.client.model.Filters;

import eu.dnetlib.data.information.oai.publisher.OaiPublisherException;

import eu.dnetlib.data.information.oai.publisher.OaiPublisherRuntimeException;

import eu.dnetlib.data.information.oai.publisher.conf.OAIConfigurationReader;

import eu.dnetlib.data.information.oai.publisher.info.MDFInfo;

import eu.dnetlib.data.oai.store.PublisherStoreDAO;

import eu.dnetlib.data.oai.store.sets.MongoSetCollection;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

public class MongoPublisherStoreDAO implements PublisherStoreDAO<MongoPublisherStore, DNetOAIMongoCursor> {

	private static final Log log = LogFactory.getLog(MongoPublisherStoreDAO.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Autowired

	private MongoPublisherCacheHelper mongoCacheHelper;

	/** Name of the collection with information about the OAI stores. **/

	private String metadataCollection;

	@Resource

	private MongoSetCollection mongoSetCollection;

	private boolean alwaysNewRecord;

	@Resource(name = "oaiConfigurationExistReader")

	private OAIConfigurationReader configuration;

	@Override

	public List<MongoPublisherStore> listPublisherStores(final String dbName) {

		return mongoCacheHelper.listPublisherStores(dbName, metadataCollection, alwaysNewRecord, mongoSetCollection);

	}

	@Override

	public MongoPublisherStore getStore(final String storeId, final String dbName) {

		return mongoCacheHelper.getStoreById(storeId, dbName, metadataCollection, alwaysNewRecord,  mongoSetCollection);

	}

	@Override

	public MongoPublisherStore getStore(final String mdfName, final String mdfInterpretation, final String mdfLayout, final String dbName) {

		return this.getStore(this.generateStoreId(mdfName, mdfInterpretation, mdfLayout), dbName);

	}

	@Override

	public MongoPublisherStore getStoreFor(final String targetMetadataPrefix, final String dbName) {

		MDFInfo info = this.configuration.getMetadataFormatInfo(targetMetadataPrefix);

		MongoPublisherStore store = this.getStore(info.getSourceFormatName(), info.getSourceFormatInterpretation(), info.getSourceFormatLayout(), dbName);

		return store;

	}

	@Override

	public MongoPublisherStore createStore(final String mdfName, final String mdfInterpretation, final String mdfLayout, final String dbName)

			throws OaiPublisherException {

		log.debug("CREATING OAI STORE");

		MongoDatabase db = this.mongoCacheHelper.getDB(dbName);

		DBObject store = createMetadataEntry(mdfName, mdfInterpretation, mdfLayout);

		MongoCollection<DBObject> metadata = db.getCollection(this.metadataCollection, DBObject.class);

		String id = (String) store.get("id");

		metadata.insertOne(store);

		log.debug("metadata insert done for store "+id);

		MongoPublisherStore theStore = this.mongoCacheHelper.getStoreById(id, dbName, this.metadataCollection, this.alwaysNewRecord, this.mongoSetCollection);

		return theStore;

	}

	@Override

	public boolean deleteStore(final String storeId, final String dbName) {

		log.debug("DELETING OAI STORE "+storeId+" -- db: "+dbName);

		this.mongoCacheHelper.deleteFromCache(storeId, dbName);

		MongoDatabase db = this.mongoCacheHelper.getDB(dbName);

		MongoCollection<DBObject> metadata = db.getCollection(this.metadataCollection, DBObject.class);

		final DBObject storeDeleted = metadata.findOneAndDelete(Filters.eq("id", storeId));

		if (storeDeleted == null) return false;

		else {

			db.getCollection(storeId).drop();

			// TODO: should drop entries related to mdPrefix served by the store we are deleting, not all of them.

			this.mongoSetCollection.dropOAISets(dbName);

			log.debug("Deleted oaistore " + storeId + ", db: " + dbName);

			return true;

		}

	}

	@Override

	public boolean deleteFromStore(final String storeId, final String dbName, final String set) {

		String setSpec = mongoSetCollection.normalizeSetSpec(set);

		log.debug(String.format("DELETING OAI STORE %s BY SET %s from db %s", storeId, setSpec, dbName));

		this.mongoCacheHelper.deleteFromCache(storeId, dbName);

		MongoDatabase db = this.mongoCacheHelper.getDB(dbName);

		MongoCollection<DBObject> metadata = db.getCollection(this.metadataCollection, DBObject.class);

		DBObject storeInfo = metadata.find(Filters.eq("id", storeId)).first();

		if (storeInfo == null) return false;

		else {

			db.getCollection(storeId).deleteMany(Filters.eq(OAIConfigurationReader.SET_FIELD, setSpec));

			this.mongoSetCollection.dropSet(dbName, setSpec);

			log.debug("Deleted set " + setSpec + " from oaistore " + storeId + ", db: " + dbName);

			return true;

		}

	}

	@Override

	public boolean deleteFromStore(final String mdfName, final String mdfInterpretation, final String mdfLayout, final String dbName, final String set) {

		return this.deleteFromStore(this.generateStoreId(mdfName, mdfInterpretation, mdfLayout), dbName, set);

	}

	@Override

	public boolean deleteStore(final String mdfName, final String mdfInterpretation, final String mdfLayout, final String dbName) {

		return this.deleteStore(this.generateStoreId(mdfName, mdfInterpretation, mdfLayout), dbName);

	}

	public void ensureIndex(final MongoPublisherStore store) {

		if (store == null) throw new OaiPublisherRuntimeException("Can't ensure index on null store");

		Thread t = new Thread() {

			@Override

			public void run() {

				store.ensureIndices();

			}

		};

		t.start();

	}

	public void ensureIndex(final String dbName) {

		List<MongoPublisherStore> stores = this.listPublisherStores(dbName);

		for (final MongoPublisherStore s : stores) {

			s.ensureIndices();

		}

	}

	private DBObject createMetadataEntry(final String mdfName, final String mdfInterpretation, final String mdfLayout) {

		DBObject info = BasicDBObjectBuilder.start("id", generateStoreId(mdfName, mdfInterpretation, mdfLayout)).append("metadataFormat", mdfName)

				.append("interpretation", mdfInterpretation).append("layout", mdfLayout).get();

		log.debug("Created DBObject for OAIStore metadata entry: "+info.toString());

		return info;

	}

	private String generateStoreId(final String mdfName, final String mdfInterpretation, final String mdfLayout) {

		return mdfName + "-" + mdfLayout + "-" + mdfInterpretation;

	}

	public String getMetadataCollection() {

		return metadataCollection;

	}

	@Required

	public void setMetadataCollection(final String metadataCollection) {

		this.metadataCollection = metadataCollection;

	}

	public OAIConfigurationReader getConfiguration() {

		return configuration;

	}

	public void setConfiguration(final OAIConfigurationReader configuration) {

		this.configuration = configuration;

	}

	public MongoSetCollection getMongoSetCollection() {

		return mongoSetCollection;

	}

	public void setMongoSetCollection(final MongoSetCollection mongoSetCollection) {

		this.mongoSetCollection = mongoSetCollection;

	}

	public boolean isAlwaysNewRecord() {

		return alwaysNewRecord;

	}

	public void setAlwaysNewRecord(final boolean alwaysNewRecord) {

		this.alwaysNewRecord = alwaysNewRecord;

	}

}

package eu.dnetlib.data.oai.store.mongo;

import java.util.Iterator;

import com.google.common.collect.Lists;

import com.mongodb.DBObject;

import com.mongodb.client.MongoCursor;

import eu.dnetlib.data.information.oai.publisher.info.RecordInfo;

import eu.dnetlib.data.oai.store.Cursor;

import eu.dnetlib.miscutils.functional.UnaryFunction;

public class DNetOAIMongoCursor implements Cursor {

	/**

	 * Underlying mongo cursor.

	 */

	private MongoCursor<DBObject> dbCursor;

	private int size = 0;

	/**

	 * Function to apply to records before delivering.

	 */

	private UnaryFunction<String, String> function;

	/**

	 * true if the RecordInfo returned by this Cursor must include the record body, false otherwise.

	 */

	private boolean bodyIncluded;

	private RecordInfoGenerator recordInfoGenerator;

	private MetadataExtractor metadataExtractor;

	private ProvenanceExtractor provenanceExtractor;

	public DNetOAIMongoCursor() {

		super();

	}

	public DNetOAIMongoCursor(final MongoCursor<DBObject> dbCursor, final boolean bodyIncluded, final RecordInfoGenerator recordInfoGenerator,

			final MetadataExtractor metadataExtractor) {

		this(dbCursor, null, bodyIncluded, recordInfoGenerator, metadataExtractor);

	}

	public DNetOAIMongoCursor(final MongoCursor<DBObject> dbCursor, final UnaryFunction<String, String> function, final boolean bodyIncluded,

			final RecordInfoGenerator recordInfoGenerator, final MetadataExtractor metadataExtractor) {

		super();

		this.dbCursor = dbCursor;

		this.size = 0;

		this.function = function;

		this.bodyIncluded = bodyIncluded;

		this.recordInfoGenerator = recordInfoGenerator;

		this.metadataExtractor = metadataExtractor;

	}

	/**

	 *

	 * {@inheritDoc}

	 */

	@Override

	public int count() {

		//I can do it because MongoCursor are always created from queries with "limit", so I do not expect the creation of the list to explode

		//to not exagerate, I'll get the size only if the current size is 0

		if (size == 0)

			size = Lists.newArrayList(dbCursor).size();

		return size;

	}

	/**

	 *

	 * {@inheritDoc}

	 *

	 * @see java.lang.Iterable#iterator()

	 */

	@Override

	public Iterator<RecordInfo> iterator() {

		return new Iterator<RecordInfo>() {

			@Override

			public boolean hasNext() {

				return dbCursor.hasNext();

			}

			@Override

			public RecordInfo next() {

				DBObject res = dbCursor.next();

				RecordInfo info = recordInfoGenerator.transformDBObject(res, bodyIncluded);

				if ((function != null) && bodyIncluded && (info != null)) {

					info.setMetadata(function.evaluate(info.getMetadata()));

				}

				return info;

			}

			@Override

			public void remove() {

				throw new UnsupportedOperationException();

			}

		};

	}

	public UnaryFunction<String, String> getFunction() {

		return function;

	}

	public void setFunction(final UnaryFunction<String, String> function) {

		this.function = function;

	}

	public MongoCursor<DBObject> getDbCursor() {

		return dbCursor;

	}

	public void setDbCursor(final MongoCursor<DBObject> dbCursor) {

		this.dbCursor = dbCursor;

	}

	@Override

	public boolean isBodyIncluded() {

		return this.bodyIncluded;

	}

	@Override

	public void setBodyIncluded(final boolean bodyIncluded) {

		this.bodyIncluded = bodyIncluded;

	}

	public RecordInfoGenerator getRecordInfoGenerator() {

		return recordInfoGenerator;

	}

	public void setRecordInfoGenerator(final RecordInfoGenerator recordInfoGenerator) {

		this.recordInfoGenerator = recordInfoGenerator;

	}

	public MetadataExtractor getMetadataExtractor() {

		return metadataExtractor;

	}

	public void setMetadataExtractor(final MetadataExtractor metadataExtractor) {

		this.metadataExtractor = metadataExtractor;

	}

	public ProvenanceExtractor getProvenanceExtractor() {

		return provenanceExtractor;

	}

	public void setProvenanceExtractor(final ProvenanceExtractor provenanceExtractor) {

		this.provenanceExtractor = provenanceExtractor;

	}

}

package eu.dnetlib.data.oai.store.mongo;

import java.util.List;

import javax.annotation.Resource;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.mongodb.DBObject;

import com.mongodb.MongoClient;

import com.mongodb.WriteConcern;

import com.mongodb.client.FindIterable;

import com.mongodb.client.MongoDatabase;

import com.mongodb.client.model.Filters;

import eu.dnetlib.data.information.oai.publisher.conf.OAIConfigurationReader;

import eu.dnetlib.data.oai.store.RecordChangeDetector;

import eu.dnetlib.data.oai.store.sets.MongoSetCollection;

import eu.dnetlib.functionality.cql.mongo.MongoCqlTranslator;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.cache.annotation.CacheEvict;

import org.springframework.cache.annotation.Cacheable;

/**

 * Created by alessia on 19/07/16.

 */

public class MongoPublisherCacheHelper {

	private static final Log log = LogFactory.getLog(MongoPublisherCacheHelper.class);

	@Autowired

	private MongoClient publisherMongoClient;

	@Resource(name = "oaiConfigurationExistReader")

	private OAIConfigurationReader configuration;

	@Resource

	private MetadataExtractor metadataExtractor;

	@Resource

	private RecordInfoGenerator recordInfoGenerator;

	@Resource

	private RecordChangeDetector recordChangeDetector;

	public  MongoDatabase getDB(final String dbName) {

		return this.publisherMongoClient.getDatabase(dbName).withWriteConcern(WriteConcern.JOURNALED);

	}

	@Cacheable(value="oaistores", key="#dbname")

	public List<MongoPublisherStore> listPublisherStores(final String dbName, final String metadataCollectionName, final boolean alwaysNewRecord, final MongoSetCollection mongoSetCollection) {

		log.info("Not using cache for listPublisherStores on "+dbName);

		final MongoDatabase db = getDB(dbName);

		final FindIterable<DBObject> stores = db.getCollection(metadataCollectionName, DBObject.class).find();

		return Lists.newArrayList(

				Iterables.transform(stores, new Function<DBObject, MongoPublisherStore>() {

					@Override

					public MongoPublisherStore apply(final DBObject storeInfo) {

						return createFromDBObject(storeInfo, db, alwaysNewRecord, mongoSetCollection);

					}

				})

		);

	}

	@Cacheable(value="oaistoresById", key="#storeId + #dbName", unless="#result == null")

	public MongoPublisherStore getStoreById(final String storeId, final String dbName, final String metadataCollectionName, final boolean alwaysNewRecord, final MongoSetCollection mongoSetCollection) {

		log.info(String.format("Not using cache for getStoreById: %s", storeId));

		DBObject storeInfo = getDB(dbName).getCollection(metadataCollectionName, DBObject.class).find(Filters.eq("id", storeId)).first();

		log.info("Got DBObject from mongo "+dbName+"."+metadataCollectionName+", id "+storeId+" is : "+storeInfo );

		return this.createFromDBObject(storeInfo, getDB(dbName), alwaysNewRecord, mongoSetCollection);

	}

	@CacheEvict(value="oaistoresById", key = "#storeId + #dbName")

	public void deleteFromCache(String storeId, String dbName){

		log.info("Evicting "+storeId+" for db "+dbName+ " from the cache");

	}

	private MongoPublisherStore createFromDBObject(final DBObject storeInfo, final MongoDatabase db, final boolean alwaysNewRecord, final MongoSetCollection mongoSetCollection) {

		if (storeInfo == null){

			log.error("cannot create MongoPublisherStore from null DBObject");

			return null;

		}

		log.debug("Creating MongoPublisherStore from DBObject "+storeInfo.toString());

		String storeId = (String) storeInfo.get("id");

		String mdFormat = (String) storeInfo.get("metadataFormat");

		String mdInterpretation = (String) storeInfo.get("interpretation");

		String mdLayout = (String) storeInfo.get("layout");

		MongoPublisherStore store = new MongoPublisherStore(storeId, mdFormat, mdInterpretation, mdLayout, db.getCollection(storeId, DBObject.class),

				this.configuration.getFields(mdFormat, mdInterpretation, mdLayout), recordInfoGenerator, this.configuration.getIdScheme(),

				this.configuration.getIdNamespace(), this.metadataExtractor, this.recordChangeDetector, alwaysNewRecord, db);

		store.setMongoSetCollection(mongoSetCollection);

		return store;