D-Net

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/cnr-mongo-mdstore/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "cnr-mongo-mdstore"}

package eu.dnetlib.data.mdstore.modular.mongodb;

import com.mongodb.BasicDBObject;

import com.mongodb.DBObject;

import com.mongodb.client.MongoCollection;

import com.mongodb.client.MongoDatabase;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDBStatus;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;

import org.junit.After;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.test.context.ContextConfiguration;

import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import static org.junit.Assert.assertEquals;

@Ignore

@RunWith(SpringJUnit4ClassRunner.class)

@ContextConfiguration(classes = ConfigurationTestConfig.class)

public class MDStoreDaoImplTest {

	@Autowired

	private MDStoreDao dao;

	@Autowired

	private MongoDatabase db;

	@Autowired

	private MDStoreTransactionManager manager;

	@After

	public void tearDown() throws MDStoreServiceException {

		dao.deleteMDStore("1");

		dao.deleteMDStore("2");

		dao.deleteMDStore("3");

		dao.deleteMDStore("4");

		dao.deleteMDStore("5");

		dao.deleteMDStore("6");

	}

	@Before

	public void setUp() throws Exception {

		dao.createMDStore("1", "F", "I", "L");

		dao.createMDStore("2", "F", "I", "L");

		dao.createMDStore("3", "F", "I", "L");

		dao.createMDStore("4", "F", "I", "L");

		dao.createMDStore("5", "F1", "I", "L");

		dao.createMDStore("6", "F1", "I", "L");

		final MongoCollection<DBObject> metadata = db.getCollection(MDStoreDaoImpl.METADATA_NAME, DBObject.class);

		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "1"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));

		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "2"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));

		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "3"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));

		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "4"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));

		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "5"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));

		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "6"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));

	}

	@Test

	public void test() throws MDStoreServiceException {

		assertEquals(40, dao.getSumOfSizes("F", "L", "I"));

		assertEquals(20, dao.getSumOfSizes("F1", "L", "I"));

		assertEquals(0, dao.getSumOfSizes("F_0", "L", "I"));

	}

	@Test

	public void getDBStatusTest() {

		final MDStoreDBStatus dbStatus = dao.getDBStatus();

		System.out.println(dbStatus);

	}

}

package eu.dnetlib.data.mdstore.modular.mongodb;

import java.io.File;

import java.io.FileInputStream;

import java.io.IOException;

import java.util.Iterator;

import java.util.UUID;

import com.mongodb.DBObject;

import com.mongodb.client.MongoDatabase;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.RecordParserFactory;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import org.apache.commons.io.IOUtils;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.core.io.ClassPathResource;

import org.springframework.test.context.ContextConfiguration;

import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

@Ignore

@RunWith(SpringJUnit4ClassRunner.class)

@ContextConfiguration(classes = ConfigurationTestConfig.class)

public class FeedSpeedTest {

	private static final int N_RECORDS = 68271;

	@Autowired

	private MongoDatabase db;

	@Autowired

	private MDStoreDao dao;

	@Autowired

	private RecordParserFactory recordParserfactory;

	@Before

	public void setup() throws MDStoreServiceException {

		dao.createMDStore("speed_test", "testFormat", "testInterpretation", "testLayout");

	}

	@Test

	public void testSpeedFromFolder() throws IOException {

		Iterable<String> iterable = new Iterable<String>() {

			private int counter = 0;

			private double last = System.currentTimeMillis();

			@Override

			public Iterator<String> iterator() {

				return new Iterator<String>() {

					@Override

					public boolean hasNext() {

						return counter < N_RECORDS;

					}

					@Override

					public String next() {

						if (counter % 10000 == 0) {

							System.out.println("10K records processed in " + (System.currentTimeMillis() - last) / 1000 + " seconds");

							last = System.currentTimeMillis();

						}

						File f = new File(String.format("/var/lib/eagle/content/EDH/HD%06d.xml", counter++));

						if (f.exists()) {

							try {

								FileInputStream fileInputStream = new FileInputStream(f);

								String s = IOUtils.toString(fileInputStream);

								fileInputStream.close();

								return s;

							} catch (Exception e) {

								return null;

							}

						} else {

							try {

								FileInputStream fileInputStream = new FileInputStream(new File("/var/lib/eagle/content/EDH/HD000001.xml"));

								String s = IOUtils.toString(fileInputStream);

								fileInputStream.close();

								return s;

							} catch (Exception e) {

								return null;

							}

						}

					}

					@Override

					public void remove() {}

				};

			}

		};

		MongoMDStore mdStore =

				new MongoMDStore(UUID.randomUUID().toString(), db.getCollection("speed_test", DBObject.class), recordParserfactory.newInstance(), true, db);

		mdStore.feed(iterable, false);

	}

	//@Ignore

	@Test

	public void testFeedSpeedFromTemplate() throws MDStoreServiceException, IOException {

		MongoMDStore mdStore =

				new MongoMDStore(UUID.randomUUID().toString(), db.getCollection("speed_test", DBObject.class), recordParserfactory.newInstance(), false, db);

		mdStore.feed(new Iterable<String>() {

			private int counter = 0;

			private double last = System.currentTimeMillis();

			private String templateRecord = IOUtils.toString(new ClassPathResource("/eu/dnetlib/data/mdstore/modular/mongodb/templateRecord.xml")

					.getInputStream());

			@Override

			public Iterator<String> iterator() {

				return new Iterator<String>() {

					@Override

					public boolean hasNext() {

						return counter < N_RECORDS;

					}

					@Override

					public String next() {

						if (counter % 10000 == 0) {

							System.out.println("10K records processed in " + (System.currentTimeMillis() - last) / 1000 + " seconds");

							last = System.currentTimeMillis();

						}

						File f = new File(String.format("/var/lib/eagle/content/EDH/HD%06d.xml", counter++));

						if (f.exists()) {

							try {

								FileInputStream fileInputStream = new FileInputStream(f);

								String s = IOUtils.toString(fileInputStream);

								fileInputStream.close();

								return s;

							} catch (Exception e) {

								return null;

							}

						} else {

							counter++;

							try {

								FileInputStream fileInputStream = new FileInputStream(new File("/var/lib/eagle/content/EDH/HD000009.xml"));

								String s = IOUtils.toString(fileInputStream);

								fileInputStream.close();

								return s;

							} catch (Exception e) {

								return null;

							}

						}

					}

					@Override

					public void remove() {}

				};

			}

		}, false);

	}

}

package eu.dnetlib.data.mdstore.modular.mongodb;

import java.io.IOException;

import java.io.InputStream;

import java.util.ArrayList;

import java.util.List;

import java.util.Map;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.MDFormatDescription;

import eu.dnetlib.data.mdstore.modular.mongodb.utils.IndexFieldRecordParser;

import eu.dnetlib.data.mdstore.modular.mongodb.utils.IndexFieldRecordParserException;

import org.apache.commons.io.IOUtils;

import org.junit.Test;

/**

 * Created by sandro on 11/29/16.

 */

public class IndexFieldParserTest {

    @Test

    public void parserTest() throws IOException, MDStoreServiceException, IndexFieldRecordParserException {

        InputStream inputStream = this.getClass().getResourceAsStream("/eu/dnetlib/data/mdstore/modular/mongodb/inputRecord.xml");

        final String inputRecord = IOUtils.toString(inputStream);

        final IndexFieldRecordParser parser = new IndexFieldRecordParser();

        final List<MDFormatDescription> mdref = new ArrayList<>();

        mdref.add(new MDFormatDescription("id", "//*[local-name()='objIdentifier']"));

        mdref.add(new MDFormatDescription("title", "//*[local-name()='title']"));

        mdref.add(new MDFormatDescription("creator", "//*[local-name()='creator']"));

        Map<String, List<String>> stringListMap = parser.parseRecord(inputRecord, mdref);

        for (String key : stringListMap.keySet()) {

            System.out.println("key = " + key);

            for (String value : stringListMap.get(key)) {

                System.out.println("\t" + value);

            }

        }

    }

}

package eu.dnetlib.data.mdstore.modular.mongodb;

import java.net.UnknownHostException;

import com.mongodb.MongoClient;

import com.mongodb.client.MongoDatabase;

import eu.dnetlib.data.mdstore.modular.RecordParserFactory;

import eu.dnetlib.data.mdstore.modular.StreamingRecordParser;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;

import org.springframework.context.annotation.Bean;

import org.springframework.context.annotation.Configuration;

@Configuration

public class ConfigurationTestConfig {

	@Bean

	public MongoDatabase db() throws UnknownHostException {

		final MongoClient mongo = new MongoClient("localhost", 27017);

		return mongo.getDatabase("mdstore_test");

	}

	@Bean

	public MDStoreTransactionManager manager() throws UnknownHostException {

		final MDStoreTransactionManagerImpl manager = new MDStoreTransactionManagerImpl();

		manager.setDb(db());

		return manager;

	}

	@Bean

	public RecordParserFactory recordParserFactory() {

		final RecordParserFactory rpfactory = new RecordParserFactory();

		rpfactory.setParserType(StreamingRecordParser.class);

		return rpfactory;

	}

	@Bean

	public MDStoreDao mdstoreDao() throws UnknownHostException {

		final MDStoreDaoImpl dao = new MDStoreDaoImpl();

		dao.setDb(db());

		dao.setRecordParserFactory(recordParserFactory());

		return dao;

	}

}

package eu.dnetlib.data.mdstore.modular.mongodb;

import java.net.UnknownHostException;

import java.util.ArrayList;

import java.util.List;

import java.util.UUID;

import com.mongodb.DB;

import com.mongodb.DBCollection;

import com.mongodb.DBCursor;

import com.mongodb.Mongo;

import com.mongodb.client.MongoDatabase;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.connector.MDStore;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;

import org.junit.Assert;

import org.junit.Ignore;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.test.context.ContextConfiguration;

import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

// TODO: reimplement tests

@Ignore

@RunWith(SpringJUnit4ClassRunner.class)

@ContextConfiguration(classes = ConfigurationTestConfig.class)

public class MDStoreTransactionManagerTest {

	@Autowired

	private MongoDatabase db;

	@Autowired

	private MDStoreTransactionManager manager;

	@Autowired

	private MDStoreDao dao;

	@Test

	public void testCreateandRetrieve() throws MDStoreServiceException {

		UUID idCreation = UUID.randomUUID();

		db.getCollection("metadataManager").drop();

		((MDStoreTransactionManagerImpl) manager).setManagerTable(null);

		String mdId = idCreation.toString() + "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";

		manager.createMDStore(mdId);

		Assert.assertNotNull(manager.startTransaction(mdId, true));

		Assert.assertNotNull(manager.startTransaction(mdId, true));

		String s = manager.getMDStoreCollection(mdId);

		Assert.assertNotNull(s);

		this.manager.dropMDStore(mdId);

		s = manager.getMDStoreCollection(mdId);

		Assert.assertNull(s);

		db.getCollection("metadataManager").drop();

		((MDStoreTransactionManagerImpl) manager).setManagerTable(null);

	}

	@Test

	public void testReadMdStore() throws MDStoreServiceException {

		UUID idCreation = UUID.randomUUID();

		String mdId = idCreation.toString() + "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";

		manager.createMDStore(mdId);

		Assert.assertNotNull(manager.readMdStore(mdId));

		Assert.assertNotNull(manager.startTransaction(mdId, true));

		Assert.assertNotNull(manager.readMdStore(mdId));

		db.getCollection("metadataManager").drop();

		((MDStoreTransactionManagerImpl) manager).setManagerTable(null);

		db.getCollection("metadataManager").drop();

		((MDStoreTransactionManagerImpl) manager).setManagerTable(null);

	}

	@Test

	public void testCommit() throws MDStoreServiceException {

		UUID idCreation = UUID.randomUUID();

		String mdId = idCreation.toString() + "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";

		manager.createMDStore(mdId);

		String idCurrent = manager.readMdStore(mdId);

		String transaction = manager.startTransaction(mdId, true);

		// Assert.assertTrue(manager.commit(transaction, mdId));

		Assert.assertNotSame(idCurrent, manager.readMdStore(mdId));

	}

	@Ignore

	@Test

	public void testDateTime() throws MDStoreServiceException, UnknownHostException {

		Mongo mongo = new Mongo("localhost", 27017);

		DB dbinput = mongo.getDB("mdstore");

		DBCollection inputCollection = dbinput.getCollection("70e07e9f-b3bf-4423-8777-b159819e0c6a");

		Assert.assertNotNull(inputCollection.findOne().get("body"));

		UUID idCreation = UUID.randomUUID();

		String mdId = idCreation.toString() + "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";

		manager.createMDStore(mdId);

		dao.createMDStore(mdId, "a", "a", "a");

		String transId = manager.startTransaction(mdId, true);

		ArrayList<String> data = new ArrayList<String>();

		DBCursor cursor = inputCollection.find();

		for (int i = 0; i < 1000; i++) {

			data.add((String) cursor.next().get("body"));

		}

		dao.getMDStore(transId).feed(data, true);

		// manager.commit(transId, mdId);

		cursor = inputCollection.find();

		transId = manager.startTransaction(mdId, false);

		data.clear();

		for (int i = 0; i < 10; i++) {

			data.add(cursor.next().get("body").toString().replace("oai:pumaoai.isti.cnr.it:", "SUUUCAAA"));

		}

		String currentId = manager.readMdStore(mdId);

		final MDStore newMdstore = dao.getMDStore(currentId);

		new Thread(() -> {

			List<String> dataInput = null;

			try {

				dataInput = newMdstore.deliver("", "", null).getResult(0, 10);

				for (int i = 0; i < 10; i++) {

					try {

						Thread.sleep(1000);

					} catch (InterruptedException e) {

						// TODO Auto-generated catch block

						e.printStackTrace();

					}

					System.out.println(dataInput.get(i));

				}

			} catch (MDStoreServiceException e) {

				e.printStackTrace();

			}

		}).start();

		dao.getMDStore(transId).feed(data, true);

		// manager.commit(transId, mdId);

	}

}

<oai:record xmlns="http://namespace.openaire.eu/"

  xmlns:dc="http://purl.org/dc/elements/1.1/"

  xmlns:dr="http://www.driver-repository.eu/namespace/dr"

  xmlns:dri="http://www.driver-repository.eu/namespace/dri"

  xmlns:oai="http://www.openarchives.org/OAI/2.0/"

  xmlns:prov="http://www.openarchives.org/OAI/2.0/provenance" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">

  <oai:header>

    <dri:objIdentifier>{id}</dri:objIdentifier>

    <dri:recordIdentifier>entityId:2533216</dri:recordIdentifier>

    <dri:dateOfCollection>2015-01-09T12:15:02.177+01:00</dri:dateOfCollection>

    <dri:repositoryId>8e17a4fe-b77d-406b-8c75-4cfa233092d2_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=</dri:repositoryId>

    <dri:datasourceprefix>DAI</dri:datasourceprefix>

  </oai:header>

  <oai:metadata>

    <eagle:eagleObject xmlns:eagle="http://www.eagle-network.eu/eagle">

      <eagle:recordSourceInfo

        landingPage="http://arachne.uni-koeln.de/entity/2533216" providerName="Arachne">entityId:2533216</eagle:recordSourceInfo>

      <eagle:editingInfo>

        <eagle:dateEdited>0000-00-00</eagle:dateEdited>

        <eagle:metadataEditor>unknown</eagle:metadataEditor>

      </eagle:editingInfo>

      <eagle:metadataIpr uri="http://creativecommons.org/licenses/by/3.0/">This file is licensed under the Creative Commons Attribution 3.0 Unported license.</eagle:metadataIpr>

      <eagle:title lang="de">CMS-XII-d009c-3_168111,02.jpg</eagle:title>

      <!-- objekt http://arachne.uni-koeln.de/entity/1160926 -->

      <eagle:description lang="de">Foto von: Siegel CMS XII D009c</eagle:description>

      <eagle:entityType>visual</eagle:entityType>

      <eagle:visualRepresentation>

        <eagle:representationType>image</eagle:representationType>

        <eagle:url>http://arachne.uni-koeln.de/entity/2533216</eagle:url>

        <eagle:thumbnail>http://arachne.dainst.org/data/image/thumbnail/2533216</eagle:thumbnail>

        <eagle:visualRepresentationIpr uri="http://creativecommons.org/licenses/by/3.0/">This file is licensed under the Creative Commons Attribution 3.0 Unported license.</eagle:visualRepresentationIpr>

        <eagle:format>jpg</eagle:format>

        <!-- objekt http://arachne.uni-koeln.de/entity/1160926 -->

        <eagle:hasArtifact>

          <eagle:artifactTitle>Siegel CMS XII D009c</eagle:artifactTitle>

          <eagle:objectType>Dreiseitiges Prisma Mallia Steatitgruppe</eagle:objectType>

          <eagle:material>weicher Stein</eagle:material>

        </eagle:hasArtifact>

      </eagle:visualRepresentation>

    </eagle:eagleObject>

  </oai:metadata>

</oai:record>

<?xml version="1.0" encoding="UTF-8"?>

<record xmlns:dc="http://purl.org/dc/elements/1.1/"

        xmlns:dr="http://www.driver-repository.eu/namespace/dr"

        xmlns:dri="http://www.driver-repository.eu/namespace/dri"

        xmlns:oaf="http://namespace.openaire.eu/oaf"

        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">

    <header xmlns="http://namespace.openaire.eu/">

        <dri:objIdentifier>od______2367::0001a50c6388e9bfcb791a924ec4b837</dri:objIdentifier>

        <dri:recordIdentifier>oai:pumaoai.isti.cnr.it:cnr.imati/cnr.ian.pv/1999-PP-018</dri:recordIdentifier>

        <dri:dateOfCollection/>

        <dri:mdFormat/>

        <dri:mdFormatInterpretation/>

        <dri:repositoryId>

            10d18b66-1d2a-4579-9adc-aa57b0821c7f_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=

        </dri:repositoryId>

        <dr:objectIdentifier/>

        <dr:dateOfCollection>2016-11-22T09:15:30.817Z</dr:dateOfCollection>

        <dr:dateOfTransformation>2016-11-22T09:18:15.904Z</dr:dateOfTransformation>

        <oaf:datasourceprefix>od______2367</oaf:datasourceprefix>

    </header>

    <metadata xmlns="http://namespace.openaire.eu/">

        <dc:title>Anisotropic mechanisms for multiphasic unipolar electrograms. Simulation studies and experimental

            recordings

        </dc:title>

        <dc:creator>Colli Franzone, Piero,</dc:creator>

        <dc:creator>Guerri, Luciano,</dc:creator>

        <dc:creator>Pennacchio, Micol,</dc:creator>

        <dc:creator>Taccardi, Bruno</dc:creator>

        <dc:date>1999-09-30</dc:date>

        <dc:description>The origin of the multiple, complex morphologies observed in

            unipolar epicardial electrograms, and their relationships with myocardial

            architecture, have not been fully elucidated. To clarify this problem we

            simulated electrograms (EGs) with a model representing the heart as an

            anisotropic bidomain with unequal anisotropy ratio, ellipsoidal ventricular

            geometry, transmural fiber rotation, epi-endocardial obliqueness of fiber

            direction and a simplified conduction system. The electrograms were compared

            with those directly recorded from the surface of isolated dog hearts immersed

            in a conducting medium. The model accurately reproduced the recorded EG

            morphologies for excitation wave fronts that reach the recording sites by

            spreading either along or across fibers.The

            origin of the multiple waves that constitute the QRS complex could be better

            understood after splitting the current sources, the potential distributions and

            the EGs into a field component (further subdivided into an axial and a conormal component) and a "reference"

            component. The split model provides an explanation of the interaction between the three-dimensional geometry

            and direction of

            propagation of a spreading wave front, the architecture of the fibers through

            which excitation is spreading, the potential distributions and the QRS wave

            forms. Because epicardial potentials, electrograms and isochrone contours can be

            computed noninvasively from body surface measurements, interpreting epicardial

            EGs in terms of intramural events may have clinical relevance.

        </dc:description>

        <dc:identifier>http://puma.isti.cnr.it/dfdownloadnew.php?ident=cnr.imati/cnr.ian.pv/1999-PP-018</dc:identifier>

        <dc:identifier>

            http://puma.isti.cnr.it/rmydownload.php?filename=cnr.imati/cnr.ian.pv/1999-PP-018/1999-PP-018_0.ps

        </dc:identifier>

        <dc:language>eng</dc:language>

        <dc:source>Preprint ercim.cnr.ian//1999-1151, 1999.</dc:source>

        <dc:subject>Electrograms, bidomain model, reference potential, cardiac potential maps, anisotropic propagation,

            source splitting

        </dc:subject>

        <dc:subject>info:eu-repo/classification/msc/78A70,65N30</dc:subject>

        <dc:rights>info:eu-repo/semantics/openAccess</dc:rights>

        <dc:type>info:eu-repo/semantics/preprint</dc:type>

        <dr:CobjCategory>0016</dr:CobjCategory>

        <dr:CobjIdentifier/>

        <oaf:dateAccepted>1999-09-30</oaf:dateAccepted>

        <oaf:collectedDatasourceid>opendoar____::2367</oaf:collectedDatasourceid>

        <oaf:accessrights>OPEN</oaf:accessrights>

        <oaf:hostedBy id="opendoar____::2367" name="PUblication MAnagement"/>

        <oaf:collectedFrom id="opendoar____::2367" name="PUblication MAnagement"/>

    </metadata>

    <about xmlns:oai="http://www.openarchives.org/OAI/2.0/" xmlns="http://namespace.openaire.eu/">

        <provenance xmlns="http://www.openarchives.org/OAI/2.0/provenance"

                    xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/provenance http://www.openarchives.org/OAI/2.0/provenance.xsd">

            <originDescription altered="true" harvestDate="2016-11-22T09:15:30.817Z">

                <baseURL>http://pumaoai.isti.cnr.it/openoai2.php</baseURL>

                <identifier>oai:pumaoai.isti.cnr.it:cnr.imati/cnr.ian.pv/1999-PP-018</identifier>

                <datestamp>1999-10-08</datestamp>

                <metadataNamespace>http://www.openarchives.org/OAI/2.0/oai_dc/</metadataNamespace>

            </originDescription>

        </provenance>

        <oaf:datainfo>

            <oaf:inferred>false</oaf:inferred>

            <oaf:deletedbyinference>false</oaf:deletedbyinference>

            <oaf:trust>0.9</oaf:trust>

            <oaf:inferenceprovenance/>

            <oaf:provenanceaction classid="sysimport:crosswalk:repository"

                                  classname="sysimport:crosswalk:repository"

                                  schemeid="dnet:provenanceActions" schemename="dnet:provenanceActions"/>

        </oaf:datainfo>

    </about>

</record>

package eu.dnetlib.mongodb;

import com.mongodb.MongoClientOptions;

import org.springframework.beans.BeansException;

import org.springframework.beans.factory.FactoryBean;

public class MongoOptionsFactory implements FactoryBean<MongoClientOptions> {

	private int connectionsPerHost;

	@Override

	public MongoClientOptions getObject() throws BeansException {

		return MongoClientOptions.builder().connectionsPerHost(connectionsPerHost).build();

	}

	@Override

	public Class<MongoClientOptions> getObjectType() {

		return MongoClientOptions.class;

	}

	@Override

	public boolean isSingleton() {

		return false;

	}

	public int getConnectionsPerHost() {

		return connectionsPerHost;

	}

	public void setConnectionsPerHost(int connectionsPerHost) {

		this.connectionsPerHost = connectionsPerHost;

	}

}

package eu.dnetlib.data.mdstore.modular.mongodb;

import java.util.ArrayList;

import java.util.Collections;

import java.util.Date;

import java.util.List;

import com.google.common.collect.Lists;

import com.mongodb.BasicDBList;

import com.mongodb.BasicDBObject;

import com.mongodb.DBObject;

import com.mongodb.WriteConcern;

import com.mongodb.client.FindIterable;

import com.mongodb.client.MongoCollection;

import com.mongodb.client.MongoDatabase;

import com.mongodb.client.MongoIterable;

import com.mongodb.client.model.Filters;

import com.mongodb.client.model.IndexOptions;

import com.mongodb.client.model.UpdateOptions;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.connector.*;

import eu.dnetlib.data.mdstore.modular.mongodb.utils.MDStoreUtils;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.bson.conversions.Bson;

import org.joda.time.DateTime;

import org.joda.time.Days;

import org.springframework.beans.factory.annotation.Required;

/**

 * The Class MDStoreTransactionManager.

 */

public class MDStoreTransactionManagerImpl implements MDStoreTransactionManager {

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(MDStoreTransactionManagerImpl.class);

	/**

	 * The table name.

	 */

	private static String TABLE_NAME = "metadataManager";

	/** The max number of concurrent transactions per mdstore. */

	private int maxTransactions = 1;

	/**

	 * The db.

	 */

	private MongoDatabase db;

	/**

	 * The manager table.

	 */

	private MongoCollection<DBObject> managerTable;

	/** The expired days. */

	private int expiredDays;

	private final IndexOptions options = new IndexOptions().background(true);

	/**

	 * Bootstrap manager.

	 */

	private void bootstrapManager() {

		log.debug("Bootstrap Manager start");

		final MongoCollection<DBObject> metadataColl = db.getCollection("metadata", DBObject.class);

		final FindIterable<DBObject> values = metadataColl.find();

		this.setManagerTable(db.getCollection(TABLE_NAME, DBObject.class));

		for (DBObject object : values) {

			final String id = (String) object.get("mdId");

			String newId = id;

			if (id.contains("_")) {

				newId = StringUtils.substringBefore(id, "_");

			}

			final BasicDBObject input = new BasicDBObject();

			input.put("mdId", id);

			input.put("currentId", newId);

			input.put("expiring", new String[] {});

			input.put("transactions", new String[] {});

			getManagerTable().insertOne(input);

			log.debug(String.format("Added %s to Metadata Manager data structure", id));

		}

		final BasicDBObject ensureIndex = new BasicDBObject();

		ensureIndex.put("mdId", 1);

		log.debug("Create index in MetadaManager ");

		this.getManagerTable().createIndex(ensureIndex, options);

	}

	/**

	 * Gets the DBObject describing an mdstore. null if there is no mdstore with the given id.

	 *

	 * @param mdstoreID

	 * @return DBObject or null

	 */

	private DBObject getMDStoreAsDBObject(String mdstoreID) throws MDStoreServiceException {

		final BasicDBObject query = new BasicDBObject();

		query.put("mdId", mdstoreID);

		final FindIterable<DBObject> it = this.getManagerTable().find(query);

		DBObject mdstoreInfo = it.first();

		return mdstoreInfo;

	}

	/**

	 * Verify consistency.

	 *

	 * @throws MDStoreServiceException

	 *             the MD store service exception

	 */

	@Override

	public void verifyConsistency() throws MDStoreServiceException {

		if (this.getManagerTable() == null) {

			if (!Lists.newArrayList(db.listCollectionNames()).contains(TABLE_NAME))

				bootstrapManager();

			else {

				this.setManagerTable(db.getCollection(TABLE_NAME, DBObject.class));

			}

		}

		if (this.getManagerTable() == null) throw new MDStoreServiceException("Something bad happen, unable to create managerTable");

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#createMDStore(java.lang.String)

	 */

	@Override

	public void createMDStore(final String mdId) throws MDStoreServiceException {

		log.debug("Creating new mdstore");

		verifyConsistency();

		String newId = mdId;

		if (mdId.contains("_")) {

			newId = StringUtils.substringBefore(mdId, "_");

		}

		final BasicDBObject instance = new BasicDBObject();

		instance.put("mdId", mdId);

		instance.put("currentId", newId);

		instance.put("expiring", new String[] {});

		getManagerTable().insertOne(instance);

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#dropMDStore(java.lang.String)

	 */

	@Override

	public void dropMDStore(final String mdId) throws MDStoreServiceException {

		verifyConsistency();

		log.debug("Droping MDStore: " + mdId);

		final BasicDBObject query = new BasicDBObject();

		query.put("mdId", mdId);

		final DBObject dropped = this.getManagerTable().findOneAndDelete(query);

		garbage();

		final String collectionName = (String) dropped.get("currentId");

		this.db.getCollection(collectionName).drop();

		this.db.getCollection("discarded-" + collectionName).drop();

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#getMDStoreCollection(java.lang.String)

	 */

	@Override

	public String getMDStoreCollection(final String mdId) throws MDStoreServiceException {

		verifyConsistency();

		DBObject mdstoreInfo = getMDStoreAsDBObject(mdId);

		if (mdstoreInfo != null)

			return (String) mdstoreInfo.get("currentId");

		else return null;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#startTransaction(java.lang.String, boolean)

	 */

	@Override

	public String startTransaction(final String mdId, final boolean refresh) throws MDStoreServiceException {

		verifyConsistency();

		log.info("Start transaction for metadata store " + mdId);

		final DBObject mdstoreInfo = getMDStoreAsDBObject(mdId);

		if (mdstoreInfo == null) throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdId);

		String idCreation = StringUtils.substringBefore(mdId, "_");

		idCreation = idCreation + "::" + System.currentTimeMillis();

		BasicDBList values;

		if (mdstoreInfo.containsField("transactions")) {

			values = (BasicDBList) mdstoreInfo.get("transactions");

			if (values.size() > getMaxTransactions())

				throw new MDStoreServiceException("Cannot create more than " + getMaxTransactions() + " transactions, found: " + values.size() + ", mdId:"

						+ mdId);

		} else {

			values = new BasicDBList();

		}

		final BasicDBObject transactionMetadata = new BasicDBObject();

		transactionMetadata.put("id", idCreation.toString());

		transactionMetadata.put("refresh", refresh);

		transactionMetadata.put("date", new Date());

		values.add(transactionMetadata);

		mdstoreInfo.put("transactions", values);

		this.getManagerTable().findOneAndReplace(new BasicDBObject("_id", mdstoreInfo.get("_id")), mdstoreInfo);

		return idCreation.toString();

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#commit(java.lang.String, java.lang.String,

	 *      eu.dnetlib.data.mdstore.modular.connector.MDStore)

	 */

	@Override

	public boolean commit(final String transactionId, final String mdstoreId, final MDStore current) throws MDStoreServiceException {

		verifyConsistency();

		final DBObject mdstoreInfo = getMDStoreAsDBObject(mdstoreId);

		if (mdstoreInfo == null) throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdstoreId);

		final BasicDBList transactions = (BasicDBList) mdstoreInfo.get("transactions");

		final DBObject transaction = findTransaction(transactions, transactionId);

		if (transaction == null) throw new MDStoreServiceException("Error, unable to find transaction with Id " + transactionId);

		final boolean refresh = (Boolean) transaction.get("refresh");

		transactions.remove(transaction);

		final String oldId = (String) mdstoreInfo.get("currentId");

		if (refresh) {

			mdstoreInfo.put("currentId", transactionId);

			final BasicDBList stillUsed = (BasicDBList) mdstoreInfo.get("expiring");

			if (stillUsed.size() == 0) {

				db.getCollection(oldId).drop();

				db.getCollection("discarded-" + oldId).drop();

			}

			log.debug("Replaced collection ");

		} else {

			log.debug("commit incremental ");

			updateIncremental(transactionId, oldId);

			db.getCollection(transactionId).drop();

			db.getCollection("discarded-" + transactionId).drop();

		}

		this.getManagerTable().findOneAndReplace(new BasicDBObject("_id", mdstoreInfo.get("_id")), mdstoreInfo);

		log.info("Committed transaction for metadata store " + mdstoreId);

		return true;

	}

	/**

	 * Find transaction.

	 *

	 * @param transactions

	 *            the transactions

	 * @param transactionId

	 *            the transaction id

	 * @return the DB object

	 */

	private DBObject findTransaction(final BasicDBList transactions, final String transactionId) {

		if (transactions.size() == 0) return null;

		for (int i = 0; i < transactions.size(); i++) {

			final BasicDBObject transaction = (BasicDBObject) transactions.get(i);

			final String id = (String) transaction.get("id");

			if (transactionId.equals(id)) return transaction;

		}

		return null;

	}

	/**

	 * Gets the db.

	 *

	 * @return the db

	 */

	public MongoDatabase getDb() {

		return db;

	}

	/**

	 * Sets the db.

	 *

	 * @param db

	 *            the db to set

	 */

	@Required

	public void setDb(final MongoDatabase db) {

		this.db = db;

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#readMdStore(java.lang.String)

	 */

	@Override

	public String readMdStore(final String mdStoreId) throws MDStoreServiceException {

		verifyConsistency();

		final DBObject mdstoreInfo = getMDStoreAsDBObject(mdStoreId);

		if (mdstoreInfo == null) throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdStoreId);

		final String currentId = (String) mdstoreInfo.get("currentId");

		final BasicDBList values = (BasicDBList) mdstoreInfo.get("expiring");

		updateMdstoreUsed(values, currentId);

		this.getManagerTable().findOneAndReplace(new BasicDBObject("_id", mdstoreInfo.get("_id")), mdstoreInfo);

		return currentId;

	}

	/**

	 * Update mdstore used.

	 *

	 * @param values

	 *            the values

	 * @param mdId

	 *            the md id

	 */

	private void updateMdstoreUsed(final BasicDBList values, final String mdId) {

		if (values.size() > 0) {

			for (int i = 0; i < values.size(); i++) {

				final DBObject obj = (DBObject) values.get(i);

				final String id = (String) obj.get("id");

				if (mdId.equals(id)) {

					obj.put("lastRead", new Date());

					return;

				}

			}

		}

		final BasicDBObject readStore = new BasicDBObject();

		readStore.put("id", mdId);

		readStore.put("lastRead", new Date());

		values.add(readStore);

	}

	/**

	 * Gets the manager table.

	 *

	 * @return the managerTable

	 */

	public MongoCollection<DBObject> getManagerTable() {

		return managerTable;

	}

	/**

	 * Sets the manager table.

	 *

	 * @param managerTable

	 *            the managerTable to set

	 */

	public void setManagerTable(final MongoCollection<DBObject> managerTable) {

		this.managerTable = managerTable;

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#getInfoForCurrentMdStore(java.lang.String)

	 */

	@Override

	public MDStoreManagerInfo getInfoForCurrentMdStore(final String mdStoreId) throws MDStoreServiceException {

		verifyConsistency();

		final DBObject mdstoreInfo = getMDStoreAsDBObject(mdStoreId);

		if (mdstoreInfo == null) throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdStoreId);

		final MDStoreManagerInfo result = new MDStoreManagerInfo();

		result.setCurrentId((String) mdstoreInfo.get("currentId"));

		result.setMdId((String) mdstoreInfo.get("mdId"));

		final BasicDBList values = (BasicDBList) mdstoreInfo.get("expiring");

		for (int i = 0; i < values.size(); i++) {

			final MDStoreExpiredInfo stillused = new MDStoreExpiredInfo();

			final DBObject value = (DBObject) values.get(i);

			stillused.setId((String) value.get("id"));

			stillused.setLastRead((Date) value.get("lastRead"));

			result.addExpiredItem(stillused);

		}

		final BasicDBList transactions = (BasicDBList) mdstoreInfo.get("transactions");

		if (transactions != null) {

			for (int i = 0; i < transactions.size(); i++) {

				final MDStoreTransactionInfo transaction = new MDStoreTransactionInfo();

				final DBObject value = (DBObject) transactions.get(i);

				final String transactionId = (String) value.get("id");

				transaction.setId(transactionId);

				transaction.setDate((Date) value.get("date"));

				transaction.setRefresh((Boolean) value.get("refresh"));

				transaction.setSize(db.getCollection(transactionId).count());

				result.addTransactionInfo(transaction);

			}

		}

		return result;

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#dropUsed(java.lang.String, java.lang.String)

	 */

	@Override

	public Boolean dropUsed(final String mdId, final String idToDrop) throws MDStoreServiceException {

		verifyConsistency();

		final DBObject mdstoreInfo = getMDStoreAsDBObject(mdId);

		if (mdstoreInfo == null) throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdId);

		return dropStore(mdstoreInfo, idToDrop, "expiring");

	}

	private boolean dropStore(DBObject mdstoreInfo, final String idToDrop, String transactionListName) throws MDStoreServiceException {

		final BasicDBList transactionList = (BasicDBList) mdstoreInfo.get(transactionListName);

		for (int i = 0; i < transactionList.size(); i++) {

			final DBObject value = (DBObject) transactionList.get(i);

			final String currentUsedId = (String) value.get("id");

			if (currentUsedId.equals(idToDrop)) {

				db.getCollection(idToDrop).drop();

				db.getCollection("discarded-" + idToDrop).drop();

				transactionList.remove(value);

				mdstoreInfo.put(transactionListName, transactionList);

				this.getManagerTable().findOneAndReplace(new BasicDBObject("_id", mdstoreInfo.get("_id")), mdstoreInfo);

				return true;

			}

		}

		throw new MDStoreServiceException("Error, unable to drop collection " + idToDrop);

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#garbage()

	 */

	@Override

	public void garbage() throws MDStoreServiceException {

		verifyConsistency();

		log.info("Start garbage collection of MdStore");

		final FindIterable<DBObject> it = this.managerTable.find();

		int totalDeleted = 0;

		for (DBObject currentObject :  it){

			if (log.isDebugEnabled()) {

				log.debug("start to check id: " + currentObject.get("currentId"));

			}

			garbageExpiring(currentObject, (String) currentObject.get("currentId"));

			garbageTransactions(currentObject, (String) currentObject.get("currentId"));

			this.getManagerTable().findOneAndReplace(new BasicDBObject("_id", currentObject.get("_id")), currentObject);

		}

		// DELETING Collection that are not in the metadataManager table

		MongoIterable<String> collections = this.db.listCollectionNames();

		for (String collection : collections) {

			if ((collection.length() > 30) && (collection.contains("discarded-") == false)) {

				final DBObject item = getMetadataObjectForCollections(collection);

				if (shouldDelete(collection, item)) {

					if (log.isDebugEnabled()) {

						log.debug("delete collection: " + collection + " from mongo");

					}

					db.getCollection(collection).drop();

					db.getCollection("discarded-" + collection).drop();

					if (log.isDebugEnabled()) {

						log.debug("delete collection: discarded-" + collection + " from mongo");

					}

				}

			}

		}

		log.info("Complete garbage collection of MdStore, total store deleted: " + totalDeleted);

	}

	private DBObject getMetadataObjectForCollections(final String collectionName) {

		if (collectionName == null) return null;

		final String postfix = "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";

		final String tmp = collectionName.contains("discarded-") == true ? StringUtils.substringAfter(collectionName, "discarded-") : collectionName;

		final String collectionNameCleaned = StringUtils.substringBefore(tmp, "::") + postfix;

		//DBObject query = QueryBuilder.start("mdId").is(collectionNameCleaned).get();

		Bson query = Filters.eq("mdId", collectionNameCleaned);

		return this.managerTable.find(query).first();

	}

	private boolean shouldDelete(final String collectionName, final DBObject metadataManagerInstance) {

		log.debug("should delete instance "+metadataManagerInstance);

		if((metadataManagerInstance== null) || (metadataManagerInstance.get("currentId")== null)){

			log.debug("the instance has not currentID");

			return true;

		}

		String currentId = (String) metadataManagerInstance.get("currentId");

		if (collectionName.equals(currentId)) return false;

		BasicDBList expiringList = (BasicDBList) metadataManagerInstance.get("expiring");

		if (findInList(expiringList, collectionName, "id") == true) return false;

		BasicDBList transactionsList = (BasicDBList) metadataManagerInstance.get("transactions");

		return findInList(transactionsList, collectionName, "id") != true;

	}

	private boolean findInList(final BasicDBList list, final String object, final String tagname) {

		if (list == null) return false;

		for (int i = 0; i < list.size(); i++) {

			DBObject currentObject = (DBObject) list.get(i);

			final String id = (String) currentObject.get(tagname);

			if (id.equals(object)) return true;

		}

		return false;

	}

	/**

	 * Delete.

	 *

	 * @param list

	 *            the list

	 * @param toRemove

	 *            the to remove

	 */

	private void delete(final BasicDBList list, final List<DBObject> toRemove) {

		for (final DBObject obj : toRemove) {

			if (log.isDebugEnabled()) {

				log.debug("deleting " + obj);

			}

			list.remove(obj);

		}

	}

	/**

	 * Garbage transactions.

	 *

	 * @param currentObject

	 *            the current object

	 * @param currentId

	 *            the current id

	 */

	private void garbageTransactions(final DBObject currentObject, final String currentId) {

		if (log.isDebugEnabled()) {

			log.debug("Start garbage transactions ");

		}

		final BasicDBList expiring = (BasicDBList) currentObject.get("transactions");

		if ((expiring == null) || (expiring.size() <= getMaxTransactions())) return;

		List<DBObject> expiringList = Lists.newArrayList();

		for (int i = 0; i < expiring.size(); i++) {

			final DBObject cobj = (DBObject) expiring.get(i);

			if (cobj != null) {

				expiringList.add((DBObject) expiring.get(i));

			}

		}

		Collections.sort(expiringList, MDStoreUtils.getComparatorOnDate());

		List<DBObject> toRemove = Lists.newArrayList();

		int i = 0;

		// We should remove the k item less recent

		// where k = numberOftotalTransaction - maxNumberOfTransaction

		// k = numberOfItemToRemove

		while (((expiringList.size() - toRemove.size()) > getMaxTransactions()) || (i < expiringList.size())) {

			DBObject currentObj = expiringList.get(i++);

			String objectId = (String) currentObj.get("id");

			if (!objectId.equals(currentId)) {

				if (log.isDebugEnabled()) {

					log.debug("delete collection: " + objectId + " from mongo");

				}

				db.getCollection(objectId).drop();

				db.getCollection("discarded-" + objectId).drop();

				if (log.isDebugEnabled()) {

					log.debug("delete collection: discarded-" + objectId + " from mongo");

				}

				toRemove.add(currentObj);

			} else {

				if (log.isDebugEnabled()) {

					log.debug("Cannot remove transaction " + objectId + " because is the currentId: " + currentId);

				}

			}

		}

		delete(expiring, toRemove);

		log.info("Deleted " + toRemove.size() + " transactions, mdStore Id:" + currentObject.get("mdId"));

	}

	/**

	 * Garbage expiring.

	 *

	 * @param currentObject

	 *            the current object

	 * @param currentId

	 *            the current id

	 */

	private void garbageExpiring(final DBObject currentObject, final String currentId) {

		if (log.isDebugEnabled()) {

			log.debug("Start to search expiring mdstores for id: " + currentObject.get("mdId"));

		}

		final BasicDBList expiring = (BasicDBList) currentObject.get("expiring");

		final List<DBObject> toRemove = Lists.newArrayList();

		if (log.isDebugEnabled()) {

			if (expiring == null) {

				log.debug("expiring list is null");

			} else {

				log.debug("expiring list size is :" + expiring.size());

			}

		}

		if ((expiring == null) || (expiring.size() == 0)) {

			log.debug("Deleted  0  expired  collections, mdStore Id:" + currentObject.get("mdId"));

			return;

		}

		for (int i = 0; i < expiring.size(); i++) {

			final DBObject currentExpiredStore = (DBObject) expiring.get(i);

			final String currentUsedId = (String) currentExpiredStore.get("id");

			final Days d = getExpiringDays(currentExpiredStore, "lastRead");

			if (log.isDebugEnabled()) {

				log.debug("the store :" + currentId + " expired since " + d.getDays() + "days ");

			}

			// DELETE the collection where the last time they was read

			// is more than 3 days ago

			if (d.getDays() > getExpiredDays()) {

				if (currentUsedId.equals(currentId) == false) {

					db.getCollection(currentUsedId).drop();

					db.getCollection("discarded-" + currentUsedId).drop();

					log.debug("deleted collection " + currentUsedId);

				}

				toRemove.add(currentExpiredStore);

			}

		}

		delete(expiring, toRemove);

		log.debug("Deleted expired " + toRemove.size() + "collections, mdStore Id:" + currentObject.get("mdId"));

	}

	/**

	 * Gets the expiring days.

	 *

	 * @param value

	 *            the value

	 * @param paramName

	 *            the param name

	 * @return the expiring days

	 */

	private Days getExpiringDays(final DBObject value, final String paramName) {

		final Date lastRead = (Date) value.get(paramName);

		final DateTime last = new DateTime(lastRead);

		final DateTime today = new DateTime();

		final Days d = Days.daysBetween(last, today);

		return d;

	}

	/**

	 * Gets the expired days.

	 *

	 * @return the expiredDays

	 */

	public int getExpiredDays() {

		if (this.expiredDays == 0) return 3;

		return expiredDays;

	}

	/**

	 * Sets the expired days.

	 *

	 * @param expiredDays

	 *            the expiredDays to set

	 */

	public void setExpiredDays(final int expiredDays) {

		this.expiredDays = expiredDays;

	}

	/**

	 * Update incremental.

	 *

	 * @param transactionId

	 *            the transaction id

	 * @param currentId

	 *            the current id

	 */

	private void updateIncremental(final String transactionId, final String currentId) {

		final MongoCollection<DBObject> transaction = db.getCollection(transactionId, DBObject.class);

		final MongoCollection<DBObject> mdstore = db.getCollection(currentId, DBObject.class);

		final FindIterable<DBObject> it = transaction.find().noCursorTimeout(true);

		for (DBObject currentObj : it) {

			BasicDBObject newObj = new BasicDBObject();

			final String id = (String) currentObj.get("id");

			final String body = (String) currentObj.get("body");

			newObj.put("id", id);

			newObj.put("body", body);

			if (StringUtils.isNotBlank(id)) {

				//setting to journaled write concern to be sure that when the write returns everything has been flushed to disk (https://docs.mongodb.org/manual/faq/developers/#when-does-mongodb-write-updates-to-disk)

				//the explicit fsync command can't be run anymore: 'Command failed with error 13: 'fsync may only be run against the admin database.'

				final MongoCollection<DBObject> mdstoreWrite = mdstore.withWriteConcern(WriteConcern.JOURNALED);

				mdstoreWrite.replaceOne(new BasicDBObject("id", id), newObj, new UpdateOptions().upsert(true));

			}

		}

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#dropTransaction(java.lang.String, java.lang.String)

	 */

	@Override

	public Boolean dropTransaction(final String mdId, final String idToDrop) throws MDStoreServiceException {

		verifyConsistency();

		final DBObject mdstoreInfo = getMDStoreAsDBObject(mdId);

		if (mdstoreInfo == null) throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdId);

		return dropStore(mdstoreInfo, idToDrop, "transactions");

	}

	public void garbageTransactionsOnStart() throws MDStoreServiceException {

		verifyConsistency();

		FindIterable<DBObject> it = this.managerTable.find();

		final List<String> currentMdStoreId = Lists.newArrayList();

		for (DBObject currentObject : it){

			currentMdStoreId.add((String) currentObject.get("currentId"));

			final BasicDBList transactions = (BasicDBList) currentObject.get("transactions");

			if ((transactions != null) && (transactions.size() > 0)) {

				for (int i = 0; i < transactions.size(); i++) {

					final DBObject currentTransactions = (DBObject) transactions.get(i);

					final String id = (String) currentTransactions.get("id");

					db.getCollection(id).drop();

					db.getCollection("discarded-" + id).drop();

					log.debug("deleted collection " + id);

				}

				currentObject.put("transactions", new BasicDBList());

				this.getManagerTable().findOneAndReplace(new BasicDBObject("_id", currentObject.get("_id")), currentObject);

			}

		}

		//DELETING ALL THE DISCARDED COLLECTION THAT DISCARDED COLLECTION OF THE CURRENT MDSTORE

		final ArrayList<String> collectionsNames = Lists.newArrayList(db.listCollectionNames());

		for (String item : collectionsNames) {

			if (item.startsWith("discarded-")) {

				final String currentCollection = StringUtils.substringAfter(item, "discarded-");

				if (!currentMdStoreId.contains(currentCollection)) {

					log.info("Deleting discarded collection :" + item);

					this.db.getCollection(item).drop();

				}

			}

		}

	}

	/**

	 * Gets the max transactions.

	 *

	 * @return the maxTransactions

	 */

	public int getMaxTransactions() {

		return maxTransactions;

	}

	/**

	 * Sets the max transactions.

	 *

	 * @param maxTransactions

	 *            the maxTransactions to set

	 */

	public void setMaxTransactions(final int maxTransactions) {

		this.maxTransactions = maxTransactions;

	}

}

package eu.dnetlib.data.mdstore.modular.mongodb;

import java.util.ArrayList;

import java.util.List;

import java.util.regex.Pattern;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.mongodb.BasicDBObject;

import com.mongodb.DBObject;

import com.mongodb.client.FindIterable;

import com.mongodb.client.MongoCollection;

import com.mongodb.client.model.Sorts;

import eu.dnetlib.enabling.resultset.ResultSet;

import eu.dnetlib.enabling.resultset.ResultSetAware;

import eu.dnetlib.enabling.resultset.ResultSetListener;

import eu.dnetlib.miscutils.maps.ConcurrentSizedMap;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.bson.conversions.Bson;

import static com.mongodb.client.model.Filters.*;

public class MongoResultSetListener implements ResultSetListener, ResultSetAware {

	private static final Log log = LogFactory.getLog(MongoResultSetListener.class);

	private ConcurrentSizedMap<Integer, String> lastKeys = new ConcurrentSizedMap<>();

	private Bson sortByIdAsc = Sorts.orderBy(Sorts.ascending("id"));

	private Function<DBObject, String> serializer;

	private MongoCollection<DBObject> collection;

	private Bson query;