Project

General

Profile

« Previous | Next » 

Revision 55308

[maven-release-plugin] copy for tag cnr-mongo-mdstore-6.0.1

View differences:

modules/cnr-mongo-mdstore/tags/cnr-mongo-mdstore-6.0.1/deploy.info
1
{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/cnr-mongo-mdstore/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "cnr-mongo-mdstore"}
modules/cnr-mongo-mdstore/tags/cnr-mongo-mdstore-6.0.1/src/test/java/eu/dnetlib/data/mdstore/modular/mongodb/MDStoreDaoImplTest.java
1
package eu.dnetlib.data.mdstore.modular.mongodb;
2

  
3
import com.mongodb.BasicDBObject;
4
import com.mongodb.DBObject;
5
import com.mongodb.client.MongoCollection;
6
import com.mongodb.client.MongoDatabase;
7
import eu.dnetlib.data.mdstore.MDStoreServiceException;
8
import eu.dnetlib.data.mdstore.modular.connector.MDStoreDBStatus;
9
import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
10
import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;
11
import org.junit.After;
12
import org.junit.Before;
13
import org.junit.Ignore;
14
import org.junit.Test;
15
import org.junit.runner.RunWith;
16
import org.springframework.beans.factory.annotation.Autowired;
17
import org.springframework.test.context.ContextConfiguration;
18
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
19

  
20
import static org.junit.Assert.assertEquals;
21

  
22
@Ignore
23
@RunWith(SpringJUnit4ClassRunner.class)
24
@ContextConfiguration(classes = ConfigurationTestConfig.class)
25
public class MDStoreDaoImplTest {
26

  
27
	@Autowired
28
	private MDStoreDao dao;
29
	@Autowired
30
	private MongoDatabase db;
31
	@Autowired
32
	private MDStoreTransactionManager manager;
33

  
34
	@After
35
	public void tearDown() throws MDStoreServiceException {
36
		dao.deleteMDStore("1");
37
		dao.deleteMDStore("2");
38
		dao.deleteMDStore("3");
39
		dao.deleteMDStore("4");
40
		dao.deleteMDStore("5");
41
		dao.deleteMDStore("6");
42
	}
43

  
44
	@Before
45
	public void setUp() throws Exception {
46
		dao.createMDStore("1", "F", "I", "L");
47
		dao.createMDStore("2", "F", "I", "L");
48
		dao.createMDStore("3", "F", "I", "L");
49
		dao.createMDStore("4", "F", "I", "L");
50
		dao.createMDStore("5", "F1", "I", "L");
51
		dao.createMDStore("6", "F1", "I", "L");
52

  
53
		final MongoCollection<DBObject> metadata = db.getCollection(MDStoreDaoImpl.METADATA_NAME, DBObject.class);
54

  
55
		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "1"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));
56
		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "2"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));
57
		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "3"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));
58
		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "4"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));
59
		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "5"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));
60
		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "6"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));
61

  
62
	}
63

  
64
	@Test
65
	public void test() throws MDStoreServiceException {
66
		assertEquals(40, dao.getSumOfSizes("F", "L", "I"));
67
		assertEquals(20, dao.getSumOfSizes("F1", "L", "I"));
68
		assertEquals(0, dao.getSumOfSizes("F_0", "L", "I"));
69
	}
70

  
71
	@Test
72
	public void getDBStatusTest() {
73
		final MDStoreDBStatus dbStatus = dao.getDBStatus();
74
		System.out.println(dbStatus);
75

  
76
	}
77

  
78
}
modules/cnr-mongo-mdstore/tags/cnr-mongo-mdstore-6.0.1/src/test/java/eu/dnetlib/data/mdstore/modular/mongodb/FeedSpeedTest.java
1
package eu.dnetlib.data.mdstore.modular.mongodb;
2

  
3
import java.io.File;
4
import java.io.FileInputStream;
5
import java.io.IOException;
6
import java.util.Iterator;
7
import java.util.UUID;
8

  
9
import com.mongodb.DBObject;
10
import com.mongodb.client.MongoDatabase;
11
import eu.dnetlib.data.mdstore.MDStoreServiceException;
12
import eu.dnetlib.data.mdstore.modular.RecordParserFactory;
13
import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
14
import org.apache.commons.io.IOUtils;
15
import org.junit.Before;
16
import org.junit.Ignore;
17
import org.junit.Test;
18
import org.junit.runner.RunWith;
19
import org.springframework.beans.factory.annotation.Autowired;
20
import org.springframework.core.io.ClassPathResource;
21
import org.springframework.test.context.ContextConfiguration;
22
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
23

  
24
@Ignore
25
@RunWith(SpringJUnit4ClassRunner.class)
26
@ContextConfiguration(classes = ConfigurationTestConfig.class)
27
public class FeedSpeedTest {
28

  
29
	private static final int N_RECORDS = 68271;
30

  
31
	@Autowired
32
	private MongoDatabase db;
33

  
34
	@Autowired
35
	private MDStoreDao dao;
36

  
37
	@Autowired
38
	private RecordParserFactory recordParserfactory;
39

  
40
	@Before
41
	public void setup() throws MDStoreServiceException {
42
		dao.createMDStore("speed_test", "testFormat", "testInterpretation", "testLayout");
43
	}
44

  
45

  
46
	@Test
47
	public void testSpeedFromFolder() throws IOException {
48
		Iterable<String> iterable = new Iterable<String>() {
49

  
50
			private int counter = 0;
51
			private double last = System.currentTimeMillis();
52

  
53
			@Override
54
			public Iterator<String> iterator() {
55
				return new Iterator<String>() {
56

  
57
					@Override
58
					public boolean hasNext() {
59
						return counter < N_RECORDS;
60
					}
61

  
62
					@Override
63
					public String next() {
64
						if (counter % 10000 == 0) {
65
							System.out.println("10K records processed in " + (System.currentTimeMillis() - last) / 1000 + " seconds");
66
							last = System.currentTimeMillis();
67
						}
68

  
69
						File f = new File(String.format("/var/lib/eagle/content/EDH/HD%06d.xml", counter++));
70
						if (f.exists()) {
71
							try {
72
								FileInputStream fileInputStream = new FileInputStream(f);
73
								String s = IOUtils.toString(fileInputStream);
74
								fileInputStream.close();
75
								return s;
76
							} catch (Exception e) {
77
								return null;
78
							}
79
						} else {
80
							try {
81
								FileInputStream fileInputStream = new FileInputStream(new File("/var/lib/eagle/content/EDH/HD000001.xml"));
82
								String s = IOUtils.toString(fileInputStream);
83
								fileInputStream.close();
84
								return s;
85
							} catch (Exception e) {
86
								return null;
87
							}
88
						}
89
					}
90

  
91
					@Override
92
					public void remove() {}
93
				};
94
			}
95
		};
96

  
97
		MongoMDStore mdStore =
98
				new MongoMDStore(UUID.randomUUID().toString(), db.getCollection("speed_test", DBObject.class), recordParserfactory.newInstance(), true, db);
99
		mdStore.feed(iterable, false);
100
	}
101

  
102
	//@Ignore
103
	@Test
104
	public void testFeedSpeedFromTemplate() throws MDStoreServiceException, IOException {
105
		MongoMDStore mdStore =
106
				new MongoMDStore(UUID.randomUUID().toString(), db.getCollection("speed_test", DBObject.class), recordParserfactory.newInstance(), false, db);
107
		mdStore.feed(new Iterable<String>() {
108

  
109
			private int counter = 0;
110
			private double last = System.currentTimeMillis();
111
			private String templateRecord = IOUtils.toString(new ClassPathResource("/eu/dnetlib/data/mdstore/modular/mongodb/templateRecord.xml")
112
					.getInputStream());
113

  
114
			@Override
115
			public Iterator<String> iterator() {
116
				return new Iterator<String>() {
117

  
118
					@Override
119
					public boolean hasNext() {
120
						return counter < N_RECORDS;
121
					}
122

  
123
					@Override
124
					public String next() {
125
						if (counter % 10000 == 0) {
126
							System.out.println("10K records processed in " + (System.currentTimeMillis() - last) / 1000 + " seconds");
127
							last = System.currentTimeMillis();
128
						}
129

  
130
						File f = new File(String.format("/var/lib/eagle/content/EDH/HD%06d.xml", counter++));
131
						if (f.exists()) {
132
							try {
133
								FileInputStream fileInputStream = new FileInputStream(f);
134
								String s = IOUtils.toString(fileInputStream);
135
								fileInputStream.close();
136
								return s;
137
							} catch (Exception e) {
138
								return null;
139
							}
140
						} else {
141
							counter++;
142
							try {
143
								FileInputStream fileInputStream = new FileInputStream(new File("/var/lib/eagle/content/EDH/HD000009.xml"));
144
								String s = IOUtils.toString(fileInputStream);
145
								fileInputStream.close();
146
								return s;
147
							} catch (Exception e) {
148
								return null;
149
							}
150
						}
151
					}
152

  
153
					@Override
154
					public void remove() {}
155
				};
156
			}
157
		}, false);
158
	}
159
}
modules/cnr-mongo-mdstore/tags/cnr-mongo-mdstore-6.0.1/src/test/java/eu/dnetlib/data/mdstore/modular/mongodb/IndexFieldParserTest.java
1
package eu.dnetlib.data.mdstore.modular.mongodb;
2

  
3
import java.io.IOException;
4
import java.io.InputStream;
5
import java.util.ArrayList;
6
import java.util.List;
7
import java.util.Map;
8

  
9
import eu.dnetlib.data.mdstore.MDStoreServiceException;
10
import eu.dnetlib.data.mdstore.modular.MDFormatDescription;
11
import eu.dnetlib.data.mdstore.modular.mongodb.utils.IndexFieldRecordParser;
12
import eu.dnetlib.data.mdstore.modular.mongodb.utils.IndexFieldRecordParserException;
13
import org.apache.commons.io.IOUtils;
14
import org.junit.Test;
15

  
16
/**
17
 * Created by sandro on 11/29/16.
18
 */
19
public class IndexFieldParserTest {
20

  
21

  
22
    @Test
23
    public void parserTest() throws IOException, MDStoreServiceException, IndexFieldRecordParserException {
24
        InputStream inputStream = this.getClass().getResourceAsStream("/eu/dnetlib/data/mdstore/modular/mongodb/inputRecord.xml");
25

  
26
        final String inputRecord = IOUtils.toString(inputStream);
27

  
28
        final IndexFieldRecordParser parser = new IndexFieldRecordParser();
29

  
30
        final List<MDFormatDescription> mdref = new ArrayList<>();
31

  
32
        mdref.add(new MDFormatDescription("id", "//*[local-name()='objIdentifier']"));
33
        mdref.add(new MDFormatDescription("title", "//*[local-name()='title']"));
34
        mdref.add(new MDFormatDescription("creator", "//*[local-name()='creator']"));
35

  
36
        Map<String, List<String>> stringListMap = parser.parseRecord(inputRecord, mdref);
37

  
38

  
39
        for (String key : stringListMap.keySet()) {
40
            System.out.println("key = " + key);
41
            for (String value : stringListMap.get(key)) {
42
                System.out.println("\t" + value);
43
            }
44

  
45
        }
46

  
47

  
48
    }
49

  
50

  
51
}
modules/cnr-mongo-mdstore/tags/cnr-mongo-mdstore-6.0.1/src/test/java/eu/dnetlib/data/mdstore/modular/mongodb/ConfigurationTestConfig.java
1
package eu.dnetlib.data.mdstore.modular.mongodb;
2

  
3
import java.net.UnknownHostException;
4

  
5
import com.mongodb.MongoClient;
6
import com.mongodb.client.MongoDatabase;
7
import eu.dnetlib.data.mdstore.modular.RecordParserFactory;
8
import eu.dnetlib.data.mdstore.modular.StreamingRecordParser;
9
import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
10
import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;
11
import org.springframework.context.annotation.Bean;
12
import org.springframework.context.annotation.Configuration;
13

  
14
@Configuration
15
public class ConfigurationTestConfig {
16

  
17
	@Bean
18
	public MongoDatabase db() throws UnknownHostException {
19
		final MongoClient mongo = new MongoClient("localhost", 27017);
20
		return mongo.getDatabase("mdstore_test");
21
	}
22

  
23
	@Bean
24
	public MDStoreTransactionManager manager() throws UnknownHostException {
25
		final MDStoreTransactionManagerImpl manager = new MDStoreTransactionManagerImpl();
26
		manager.setDb(db());
27
		return manager;
28
	}
29

  
30
	@Bean
31
	public RecordParserFactory recordParserFactory() {
32
		final RecordParserFactory rpfactory = new RecordParserFactory();
33
		rpfactory.setParserType(StreamingRecordParser.class);
34
		return rpfactory;
35
	}
36

  
37
	@Bean
38
	public MDStoreDao mdstoreDao() throws UnknownHostException {
39
		final MDStoreDaoImpl dao = new MDStoreDaoImpl();
40
		dao.setDb(db());
41
		dao.setRecordParserFactory(recordParserFactory());
42
		return dao;
43
	}
44

  
45
}
modules/cnr-mongo-mdstore/tags/cnr-mongo-mdstore-6.0.1/src/test/java/eu/dnetlib/data/mdstore/modular/mongodb/MDStoreTransactionManagerTest.java
1
package eu.dnetlib.data.mdstore.modular.mongodb;
2

  
3
import java.net.UnknownHostException;
4
import java.util.ArrayList;
5
import java.util.List;
6
import java.util.UUID;
7

  
8
import com.mongodb.DB;
9
import com.mongodb.DBCollection;
10
import com.mongodb.DBCursor;
11
import com.mongodb.Mongo;
12
import com.mongodb.client.MongoDatabase;
13
import eu.dnetlib.data.mdstore.MDStoreServiceException;
14
import eu.dnetlib.data.mdstore.modular.connector.MDStore;
15
import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
16
import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;
17
import org.junit.Assert;
18
import org.junit.Ignore;
19
import org.junit.Test;
20
import org.junit.runner.RunWith;
21
import org.springframework.beans.factory.annotation.Autowired;
22
import org.springframework.test.context.ContextConfiguration;
23
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
24

  
25
// TODO: reimplement tests
26

  
27
@Ignore
28
@RunWith(SpringJUnit4ClassRunner.class)
29
@ContextConfiguration(classes = ConfigurationTestConfig.class)
30
public class MDStoreTransactionManagerTest {
31

  
32
	@Autowired
33
	private MongoDatabase db;
34

  
35
	@Autowired
36
	private MDStoreTransactionManager manager;
37

  
38
	@Autowired
39
	private MDStoreDao dao;
40

  
41
	@Test
42
	public void testCreateandRetrieve() throws MDStoreServiceException {
43
		UUID idCreation = UUID.randomUUID();
44
		db.getCollection("metadataManager").drop();
45
		((MDStoreTransactionManagerImpl) manager).setManagerTable(null);
46

  
47
		String mdId = idCreation.toString() + "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";
48
		manager.createMDStore(mdId);
49

  
50
		Assert.assertNotNull(manager.startTransaction(mdId, true));
51
		Assert.assertNotNull(manager.startTransaction(mdId, true));
52

  
53
		String s = manager.getMDStoreCollection(mdId);
54
		Assert.assertNotNull(s);
55
		this.manager.dropMDStore(mdId);
56
		s = manager.getMDStoreCollection(mdId);
57
		Assert.assertNull(s);
58
		db.getCollection("metadataManager").drop();
59
		((MDStoreTransactionManagerImpl) manager).setManagerTable(null);
60
	}
61

  
62
	@Test
63
	public void testReadMdStore() throws MDStoreServiceException {
64
		UUID idCreation = UUID.randomUUID();
65
		String mdId = idCreation.toString() + "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";
66
		manager.createMDStore(mdId);
67
		Assert.assertNotNull(manager.readMdStore(mdId));
68
		Assert.assertNotNull(manager.startTransaction(mdId, true));
69
		Assert.assertNotNull(manager.readMdStore(mdId));
70
		db.getCollection("metadataManager").drop();
71
		((MDStoreTransactionManagerImpl) manager).setManagerTable(null);
72
		db.getCollection("metadataManager").drop();
73
		((MDStoreTransactionManagerImpl) manager).setManagerTable(null);
74

  
75
	}
76

  
77
	@Test
78
	public void testCommit() throws MDStoreServiceException {
79
		UUID idCreation = UUID.randomUUID();
80
		String mdId = idCreation.toString() + "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";
81
		manager.createMDStore(mdId);
82
		String idCurrent = manager.readMdStore(mdId);
83
		String transaction = manager.startTransaction(mdId, true);
84
		// Assert.assertTrue(manager.commit(transaction, mdId));
85
		Assert.assertNotSame(idCurrent, manager.readMdStore(mdId));
86

  
87
	}
88

  
89
	@Ignore
90
	@Test
91
	public void testDateTime() throws MDStoreServiceException, UnknownHostException {
92
		Mongo mongo = new Mongo("localhost", 27017);
93
		DB dbinput = mongo.getDB("mdstore");
94
		DBCollection inputCollection = dbinput.getCollection("70e07e9f-b3bf-4423-8777-b159819e0c6a");
95

  
96
		Assert.assertNotNull(inputCollection.findOne().get("body"));
97
		UUID idCreation = UUID.randomUUID();
98

  
99
		String mdId = idCreation.toString() + "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";
100

  
101
		manager.createMDStore(mdId);
102
		dao.createMDStore(mdId, "a", "a", "a");
103

  
104
		String transId = manager.startTransaction(mdId, true);
105

  
106
		ArrayList<String> data = new ArrayList<String>();
107
		DBCursor cursor = inputCollection.find();
108

  
109
		for (int i = 0; i < 1000; i++) {
110
			data.add((String) cursor.next().get("body"));
111
		}
112
		dao.getMDStore(transId).feed(data, true);
113
		// manager.commit(transId, mdId);
114

  
115
		cursor = inputCollection.find();
116
		transId = manager.startTransaction(mdId, false);
117

  
118
		data.clear();
119
		for (int i = 0; i < 10; i++) {
120
			data.add(cursor.next().get("body").toString().replace("oai:pumaoai.isti.cnr.it:", "SUUUCAAA"));
121
		}
122

  
123
		String currentId = manager.readMdStore(mdId);
124

  
125
		final MDStore newMdstore = dao.getMDStore(currentId);
126

  
127
		new Thread(() -> {
128
			List<String> dataInput = null;
129
			try {
130
				dataInput = newMdstore.deliver("", "", null).getResult(0, 10);
131
				for (int i = 0; i < 10; i++) {
132
					try {
133
						Thread.sleep(1000);
134
					} catch (InterruptedException e) {
135
						// TODO Auto-generated catch block
136
						e.printStackTrace();
137
					}
138
					System.out.println(dataInput.get(i));
139
				}
140
			} catch (MDStoreServiceException e) {
141
				e.printStackTrace();
142
			}
143
		}).start();
144

  
145
		dao.getMDStore(transId).feed(data, true);
146
		// manager.commit(transId, mdId);
147

  
148
	}
149
}
modules/cnr-mongo-mdstore/tags/cnr-mongo-mdstore-6.0.1/src/test/resources/eu/dnetlib/data/mdstore/modular/mongodb/templateRecord.xml
1
<oai:record xmlns="http://namespace.openaire.eu/"
2
  xmlns:dc="http://purl.org/dc/elements/1.1/"
3
  xmlns:dr="http://www.driver-repository.eu/namespace/dr"
4
  xmlns:dri="http://www.driver-repository.eu/namespace/dri"
5
  xmlns:oai="http://www.openarchives.org/OAI/2.0/"
6
  xmlns:prov="http://www.openarchives.org/OAI/2.0/provenance" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
7
  <oai:header>
8
    <dri:objIdentifier>{id}</dri:objIdentifier>
9
    <dri:recordIdentifier>entityId:2533216</dri:recordIdentifier>
10
    <dri:dateOfCollection>2015-01-09T12:15:02.177+01:00</dri:dateOfCollection>
11
    <dri:repositoryId>8e17a4fe-b77d-406b-8c75-4cfa233092d2_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=</dri:repositoryId>
12
    <dri:datasourceprefix>DAI</dri:datasourceprefix>
13
  </oai:header>
14
  <oai:metadata>
15
    <eagle:eagleObject xmlns:eagle="http://www.eagle-network.eu/eagle">
16
      <eagle:recordSourceInfo
17
        landingPage="http://arachne.uni-koeln.de/entity/2533216" providerName="Arachne">entityId:2533216</eagle:recordSourceInfo>
18
      <eagle:editingInfo>
19
        <eagle:dateEdited>0000-00-00</eagle:dateEdited>
20
        <eagle:metadataEditor>unknown</eagle:metadataEditor>
21
      </eagle:editingInfo>
22
      <eagle:metadataIpr uri="http://creativecommons.org/licenses/by/3.0/">This file is licensed under the Creative Commons Attribution 3.0 Unported license.</eagle:metadataIpr>
23
      <eagle:title lang="de">CMS-XII-d009c-3_168111,02.jpg</eagle:title>
24
      <!-- objekt http://arachne.uni-koeln.de/entity/1160926 -->
25
      <eagle:description lang="de">Foto von: Siegel CMS XII D009c</eagle:description>
26
      <eagle:entityType>visual</eagle:entityType>
27
      <eagle:visualRepresentation>
28
        <eagle:representationType>image</eagle:representationType>
29
        <eagle:url>http://arachne.uni-koeln.de/entity/2533216</eagle:url>
30
        <eagle:thumbnail>http://arachne.dainst.org/data/image/thumbnail/2533216</eagle:thumbnail>
31
        <eagle:visualRepresentationIpr uri="http://creativecommons.org/licenses/by/3.0/">This file is licensed under the Creative Commons Attribution 3.0 Unported license.</eagle:visualRepresentationIpr>
32
        <eagle:format>jpg</eagle:format>
33
        <!-- objekt http://arachne.uni-koeln.de/entity/1160926 -->
34
        <eagle:hasArtifact>
35
          <eagle:artifactTitle>Siegel CMS XII D009c</eagle:artifactTitle>
36
          <eagle:objectType>Dreiseitiges Prisma Mallia Steatitgruppe</eagle:objectType>
37
          <eagle:material>weicher Stein</eagle:material>
38
        </eagle:hasArtifact>
39
      </eagle:visualRepresentation>
40
    </eagle:eagleObject>
41
  </oai:metadata>
42
</oai:record>
modules/cnr-mongo-mdstore/tags/cnr-mongo-mdstore-6.0.1/src/test/resources/eu/dnetlib/data/mdstore/modular/mongodb/inputRecord.xml
1
<?xml version="1.0" encoding="UTF-8"?>
2
<record xmlns:dc="http://purl.org/dc/elements/1.1/"
3
        xmlns:dr="http://www.driver-repository.eu/namespace/dr"
4
        xmlns:dri="http://www.driver-repository.eu/namespace/dri"
5
        xmlns:oaf="http://namespace.openaire.eu/oaf"
6
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
7
    <header xmlns="http://namespace.openaire.eu/">
8
        <dri:objIdentifier>od______2367::0001a50c6388e9bfcb791a924ec4b837</dri:objIdentifier>
9
        <dri:recordIdentifier>oai:pumaoai.isti.cnr.it:cnr.imati/cnr.ian.pv/1999-PP-018</dri:recordIdentifier>
10
        <dri:dateOfCollection/>
11
        <dri:mdFormat/>
12
        <dri:mdFormatInterpretation/>
13
        <dri:repositoryId>
14
            10d18b66-1d2a-4579-9adc-aa57b0821c7f_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
15
        </dri:repositoryId>
16
        <dr:objectIdentifier/>
17
        <dr:dateOfCollection>2016-11-22T09:15:30.817Z</dr:dateOfCollection>
18
        <dr:dateOfTransformation>2016-11-22T09:18:15.904Z</dr:dateOfTransformation>
19
        <oaf:datasourceprefix>od______2367</oaf:datasourceprefix>
20
    </header>
21
    <metadata xmlns="http://namespace.openaire.eu/">
22
        <dc:title>Anisotropic mechanisms for multiphasic unipolar electrograms. Simulation studies and experimental
23
            recordings
24
        </dc:title>
25
        <dc:creator>Colli Franzone, Piero,</dc:creator>
26
        <dc:creator>Guerri, Luciano,</dc:creator>
27
        <dc:creator>Pennacchio, Micol,</dc:creator>
28
        <dc:creator>Taccardi, Bruno</dc:creator>
29
        <dc:date>1999-09-30</dc:date>
30
        <dc:description>The origin of the multiple, complex morphologies observed in
31
            unipolar epicardial electrograms, and their relationships with myocardial
32
            architecture, have not been fully elucidated. To clarify this problem we
33
            simulated electrograms (EGs) with a model representing the heart as an
34
            anisotropic bidomain with unequal anisotropy ratio, ellipsoidal ventricular
35
            geometry, transmural fiber rotation, epi-endocardial obliqueness of fiber
36
            direction and a simplified conduction system. The electrograms were compared
37
            with those directly recorded from the surface of isolated dog hearts immersed
38
            in a conducting medium. The model accurately reproduced the recorded EG
39
            morphologies for excitation wave fronts that reach the recording sites by
40
            spreading either along or across fibers.The
41
            origin of the multiple waves that constitute the QRS complex could be better
42
            understood after splitting the current sources, the potential distributions and
43
            the EGs into a field component (further subdivided into an axial and a conormal component) and a "reference"
44
            component. The split model provides an explanation of the interaction between the three-dimensional geometry
45
            and direction of
46
            propagation of a spreading wave front, the architecture of the fibers through
47
            which excitation is spreading, the potential distributions and the QRS wave
48
            forms. Because epicardial potentials, electrograms and isochrone contours can be
49
            computed noninvasively from body surface measurements, interpreting epicardial
50
            EGs in terms of intramural events may have clinical relevance.
51
        </dc:description>
52
        <dc:identifier>http://puma.isti.cnr.it/dfdownloadnew.php?ident=cnr.imati/cnr.ian.pv/1999-PP-018</dc:identifier>
53
        <dc:identifier>
54
            http://puma.isti.cnr.it/rmydownload.php?filename=cnr.imati/cnr.ian.pv/1999-PP-018/1999-PP-018_0.ps
55
        </dc:identifier>
56
        <dc:language>eng</dc:language>
57
        <dc:source>Preprint ercim.cnr.ian//1999-1151, 1999.</dc:source>
58
        <dc:subject>Electrograms, bidomain model, reference potential, cardiac potential maps, anisotropic propagation,
59
            source splitting
60
        </dc:subject>
61
        <dc:subject>info:eu-repo/classification/msc/78A70,65N30</dc:subject>
62
        <dc:rights>info:eu-repo/semantics/openAccess</dc:rights>
63
        <dc:type>info:eu-repo/semantics/preprint</dc:type>
64
        <dr:CobjCategory>0016</dr:CobjCategory>
65
        <dr:CobjIdentifier/>
66
        <oaf:dateAccepted>1999-09-30</oaf:dateAccepted>
67
        <oaf:collectedDatasourceid>opendoar____::2367</oaf:collectedDatasourceid>
68
        <oaf:accessrights>OPEN</oaf:accessrights>
69
        <oaf:hostedBy id="opendoar____::2367" name="PUblication MAnagement"/>
70
        <oaf:collectedFrom id="opendoar____::2367" name="PUblication MAnagement"/>
71
    </metadata>
72
    <about xmlns:oai="http://www.openarchives.org/OAI/2.0/" xmlns="http://namespace.openaire.eu/">
73
        <provenance xmlns="http://www.openarchives.org/OAI/2.0/provenance"
74
                    xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/provenance http://www.openarchives.org/OAI/2.0/provenance.xsd">
75
            <originDescription altered="true" harvestDate="2016-11-22T09:15:30.817Z">
76
                <baseURL>http://pumaoai.isti.cnr.it/openoai2.php</baseURL>
77
                <identifier>oai:pumaoai.isti.cnr.it:cnr.imati/cnr.ian.pv/1999-PP-018</identifier>
78
                <datestamp>1999-10-08</datestamp>
79
                <metadataNamespace>http://www.openarchives.org/OAI/2.0/oai_dc/</metadataNamespace>
80
            </originDescription>
81
        </provenance>
82
        <oaf:datainfo>
83
            <oaf:inferred>false</oaf:inferred>
84
            <oaf:deletedbyinference>false</oaf:deletedbyinference>
85
            <oaf:trust>0.9</oaf:trust>
86
            <oaf:inferenceprovenance/>
87
            <oaf:provenanceaction classid="sysimport:crosswalk:repository"
88
                                  classname="sysimport:crosswalk:repository"
89
                                  schemeid="dnet:provenanceActions" schemename="dnet:provenanceActions"/>
90
        </oaf:datainfo>
91
    </about>
92
</record>
modules/cnr-mongo-mdstore/tags/cnr-mongo-mdstore-6.0.1/src/main/java/eu/dnetlib/mongodb/MongoOptionsFactory.java
1
package eu.dnetlib.mongodb;
2

  
3
import com.mongodb.MongoClientOptions;
4
import org.springframework.beans.BeansException;
5
import org.springframework.beans.factory.FactoryBean;
6

  
7
public class MongoOptionsFactory implements FactoryBean<MongoClientOptions> {
8
	private int connectionsPerHost;
9

  
10
	@Override
11
	public MongoClientOptions getObject() throws BeansException {
12
		return MongoClientOptions.builder().connectionsPerHost(connectionsPerHost).build();
13
	}
14

  
15
	@Override
16
	public Class<MongoClientOptions> getObjectType() {
17
		return MongoClientOptions.class;
18
	}
19

  
20
	@Override
21
	public boolean isSingleton() {
22
		return false;
23
	}
24

  
25
	public int getConnectionsPerHost() {
26
		return connectionsPerHost;
27
	}
28

  
29
	public void setConnectionsPerHost(int connectionsPerHost) {
30
		this.connectionsPerHost = connectionsPerHost;
31
	}
32

  
33
}
0 34

  
modules/cnr-mongo-mdstore/tags/cnr-mongo-mdstore-6.0.1/src/main/java/eu/dnetlib/data/mdstore/modular/mongodb/MDStoreTransactionManagerImpl.java
1
package eu.dnetlib.data.mdstore.modular.mongodb;
2

  
3
import java.util.ArrayList;
4
import java.util.Date;
5
import java.util.List;
6

  
7
import com.google.common.collect.Lists;
8
import com.mongodb.BasicDBList;
9
import com.mongodb.BasicDBObject;
10
import com.mongodb.DBObject;
11
import com.mongodb.WriteConcern;
12
import com.mongodb.client.FindIterable;
13
import com.mongodb.client.MongoCollection;
14
import com.mongodb.client.MongoDatabase;
15
import com.mongodb.client.MongoIterable;
16
import com.mongodb.client.model.Filters;
17
import com.mongodb.client.model.IndexOptions;
18
import com.mongodb.client.model.UpdateOptions;
19
import eu.dnetlib.data.mdstore.MDStoreServiceException;
20
import eu.dnetlib.data.mdstore.modular.connector.*;
21
import eu.dnetlib.data.mdstore.modular.mongodb.utils.MDStoreUtils;
22
import org.apache.commons.lang3.StringUtils;
23
import org.apache.commons.logging.Log;
24
import org.apache.commons.logging.LogFactory;
25
import org.bson.conversions.Bson;
26
import org.joda.time.DateTime;
27
import org.joda.time.Days;
28
import org.springframework.beans.factory.annotation.Required;
29

  
30
/**
31
 * The Class MDStoreTransactionManager.
32
 */
33
public class MDStoreTransactionManagerImpl implements MDStoreTransactionManager {
34

  
35
	/** The Constant log. */
36
	private static final Log log = LogFactory.getLog(MDStoreTransactionManagerImpl.class);
37

  
38
	/**
39
	 * The table name.
40
	 */
41
	private static String TABLE_NAME = "metadataManager";
42

  
43
	/** The max number of concurrent transactions per mdstore. */
44
	private int maxTransactions = 1;
45

  
46
	/**
47
	 * The db.
48
	 */
49
	private MongoDatabase db;
50

  
51
	/**
52
	 * The manager table.
53
	 */
54
	private MongoCollection<DBObject> managerTable;
55

  
56
	/** The expired days. */
57
	private int expiredDays;
58

  
59
	private final IndexOptions options = new IndexOptions().background(true);
60

  
61
	/**
62
	 * Bootstrap manager.
63
	 */
64
	private void bootstrapManager() {
65
		log.debug("Bootstrap Manager start");
66
		final MongoCollection<DBObject> metadataColl = db.getCollection("metadata", DBObject.class);
67
		final FindIterable<DBObject> values = metadataColl.find();
68
		this.setManagerTable(db.getCollection(TABLE_NAME, DBObject.class));
69
		for (DBObject object : values) {
70
			final String id = (String) object.get("mdId");
71
			String newId = id;
72
			if (id.contains("_")) {
73
				newId = StringUtils.substringBefore(id, "_");
74
			}
75
			final BasicDBObject input = new BasicDBObject();
76
			input.put("mdId", id);
77
			input.put("currentId", newId);
78
			input.put("expiring", new String[] {});
79
			input.put("transactions", new String[] {});
80
			getManagerTable().insertOne(input);
81
			log.debug(String.format("Added %s to Metadata Manager data structure", id));
82

  
83
		}
84
		final BasicDBObject ensureIndex = new BasicDBObject();
85
		ensureIndex.put("mdId", 1);
86
		log.debug("Create index in MetadaManager ");
87
		this.getManagerTable().createIndex(ensureIndex, options);
88
	}
89

  
90
	/**
91
	 * Gets the DBObject describing an mdstore. null if there is no mdstore with the given id.
92
	 *
93
	 * @param mdstoreID the mdStore identifier
94
	 * @return DBObject or null
95
	 */
96
	private DBObject getMDStoreAsDBObject(final String mdstoreID) throws MDStoreServiceException {
97
		final BasicDBObject query = new BasicDBObject();
98
		query.put("mdId", mdstoreID);
99
		final FindIterable<DBObject> it = this.getManagerTable().find(query);
100
		return it.first();
101
	}
102

  
103
	/**
104
	 * Verify consistency.
105
	 *
106
	 * @throws MDStoreServiceException
107
	 *             the MD store service exception
108
	 */
109
	@Override
110
	public void verifyConsistency() throws MDStoreServiceException {
111
		if (this.getManagerTable() == null) {
112
			if (!Lists.newArrayList(db.listCollectionNames()).contains(TABLE_NAME))
113
				bootstrapManager();
114
			else {
115
				this.setManagerTable(db.getCollection(TABLE_NAME, DBObject.class));
116
			}
117
		}
118
		if (this.getManagerTable() == null) throw new MDStoreServiceException("Something bad happen, unable to create managerTable");
119
	}
120

  
121
	/**
122
	 * {@inheritDoc}
123
	 *
124
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#createMDStore(java.lang.String)
125
	 */
126
	@Override
127
	public void createMDStore(final String mdId) throws MDStoreServiceException {
128
		log.debug("Creating new mdstore");
129
		verifyConsistency();
130
		String newId = mdId;
131
		if (mdId.contains("_")) {
132
			newId = StringUtils.substringBefore(mdId, "_");
133
		}
134
		final BasicDBObject instance = new BasicDBObject();
135
		instance.put("mdId", mdId);
136
		instance.put("currentId", newId);
137
		instance.put("expiring", new String[] {});
138
		getManagerTable().insertOne(instance);
139
	}
140

  
141
	/**
142
	 * {@inheritDoc}
143
	 *
144
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#dropMDStore(java.lang.String)
145
	 */
146
	@Override
147
	public void dropMDStore(final String mdId) throws MDStoreServiceException {
148
		verifyConsistency();
149
		log.debug("Droping MDStore: " + mdId);
150
		final BasicDBObject query = new BasicDBObject();
151
		query.put("mdId", mdId);
152
		final DBObject dropped = this.getManagerTable().findOneAndDelete(query);
153
		garbage();
154
		final String collectionName = (String) dropped.get("currentId");
155
		this.db.getCollection(collectionName).drop();
156
		this.db.getCollection("discarded-" + collectionName).drop();
157
	}
158

  
159
	/**
160
	 * {@inheritDoc}
161
	 *
162
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#getMDStoreCollection(java.lang.String)
163
	 */
164
	@Override
165
	public String getMDStoreCollection(final String mdId) throws MDStoreServiceException {
166
		verifyConsistency();
167
		DBObject mdstoreInfo = getMDStoreAsDBObject(mdId);
168
		if (mdstoreInfo != null)
169
			return (String) mdstoreInfo.get("currentId");
170
		else return null;
171
	}
172

  
173
	/**
174
	 * {@inheritDoc}
175
	 *
176
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#startTransaction(java.lang.String, boolean)
177
	 */
178
	@Override
179
	public String startTransaction(final String mdId, final boolean refresh) throws MDStoreServiceException {
180
		verifyConsistency();
181
		log.info("Start transaction for metadata store " + mdId);
182
		final DBObject mdstoreInfo = getMDStoreAsDBObject(mdId);
183
		if (mdstoreInfo == null) throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdId);
184
		String idCreation = StringUtils.substringBefore(mdId, "_");
185
		idCreation = idCreation + "::" + System.currentTimeMillis();
186

  
187
		BasicDBList values;
188
		if (mdstoreInfo.containsField("transactions")) {
189
			values = (BasicDBList) mdstoreInfo.get("transactions");
190
			if (values.size() > getMaxTransactions())
191
				throw new MDStoreServiceException("Cannot create more than " + getMaxTransactions() + " transactions, found: " + values.size() + ", mdId:"
192
						+ mdId);
193
		} else {
194
			values = new BasicDBList();
195
		}
196
		final BasicDBObject transactionMetadata = new BasicDBObject();
197
		transactionMetadata.put("id", idCreation);
198
		transactionMetadata.put("refresh", refresh);
199
		transactionMetadata.put("date", new Date());
200
		values.add(transactionMetadata);
201
		mdstoreInfo.put("transactions", values);
202
		this.getManagerTable().findOneAndReplace(new BasicDBObject("_id", mdstoreInfo.get("_id")), mdstoreInfo);
203
		return idCreation;
204
	}
205

  
206
	/**
207
	 * {@inheritDoc}
208
	 *
209
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#commit(java.lang.String, java.lang.String,
210
	 *      eu.dnetlib.data.mdstore.modular.connector.MDStore)
211
	 */
212
	@Override
213
	public boolean commit(final String transactionId, final String mdstoreId, final MDStore current) throws MDStoreServiceException {
214
		verifyConsistency();
215
		final DBObject mdstoreInfo = getMDStoreAsDBObject(mdstoreId);
216
		if (mdstoreInfo == null) throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdstoreId);
217
		final BasicDBList transactions = (BasicDBList) mdstoreInfo.get("transactions");
218
		final DBObject transaction = findTransaction(transactions, transactionId);
219
		if (transaction == null) throw new MDStoreServiceException("Error, unable to find transaction with Id " + transactionId);
220
		final boolean refresh = (Boolean) transaction.get("refresh");
221
		transactions.remove(transaction);
222
		final String oldId = (String) mdstoreInfo.get("currentId");
223
		if (refresh) {
224
			mdstoreInfo.put("currentId", transactionId);
225
			final BasicDBList stillUsed = (BasicDBList) mdstoreInfo.get("expiring");
226
			if (stillUsed.size() == 0) {
227
				db.getCollection(oldId).drop();
228
				db.getCollection("discarded-" + oldId).drop();
229
			}
230
			log.debug("Replaced collection ");
231
		} else {
232
			log.debug("commit incremental ");
233
			updateIncremental(transactionId, oldId);
234
			db.getCollection(transactionId).drop();
235
			db.getCollection("discarded-" + transactionId).drop();
236
		}
237
		this.getManagerTable().findOneAndReplace(new BasicDBObject("_id", mdstoreInfo.get("_id")), mdstoreInfo);
238

  
239
		log.info("Committed transaction for metadata store " + mdstoreId);
240
		return true;
241
	}
242

  
243
	/**
244
	 * Find transaction.
245
	 *
246
	 * @param transactions
247
	 *            the transactions
248
	 * @param transactionId
249
	 *            the transaction id
250
	 * @return the DB object
251
	 */
252
	private DBObject findTransaction(final BasicDBList transactions, final String transactionId) {
253
		if (transactions.size() == 0) return null;
254
		for (Object tx : transactions) {
255
			final BasicDBObject transaction = (BasicDBObject) tx;
256
			final String id = (String) transaction.get("id");
257
			if (transactionId.equals(id)) return transaction;
258
		}
259
		return null;
260

  
261
	}
262

  
263
	/**
264
	 * Gets the db.
265
	 *
266
	 * @return the db
267
	 */
268
	public MongoDatabase getDb() {
269
		return db;
270
	}
271

  
272
	/**
273
	 * Sets the db.
274
	 *
275
	 * @param db
276
	 *            the db to set
277
	 */
278
	@Required
279
	public void setDb(final MongoDatabase db) {
280
		this.db = db;
281
	}
282

  
283
	/**
284
	 * {@inheritDoc}
285
	 *
286
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#readMdStore(java.lang.String)
287
	 */
288
	@Override
289
	public String readMdStore(final String mdStoreId) throws MDStoreServiceException {
290
		verifyConsistency();
291
		final DBObject mdstoreInfo = getMDStoreAsDBObject(mdStoreId);
292
		if (mdstoreInfo == null) throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdStoreId);
293
		final String currentId = (String) mdstoreInfo.get("currentId");
294
		final BasicDBList values = (BasicDBList) mdstoreInfo.get("expiring");
295
		updateMdstoreUsed(values, currentId);
296
		this.getManagerTable().findOneAndReplace(new BasicDBObject("_id", mdstoreInfo.get("_id")), mdstoreInfo);
297
		return currentId;
298

  
299
	}
300

  
301

  
302
	/**
303
	 * Update mdstore used.
304
	 *
305
	 * @param values
306
	 *            the values
307
	 * @param mdId
308
	 *            the md id
309
	 */
310
	private void updateMdstoreUsed(final BasicDBList values, final String mdId) {
311
		if (values.size() > 0) {
312
			for (Object value : values) {
313
				final DBObject obj = (DBObject) value;
314
				final String id = (String) obj.get("id");
315
				if (mdId.equals(id)) {
316
					obj.put("lastRead", new Date());
317
					return;
318
				}
319
			}
320
		}
321
		final BasicDBObject readStore = new BasicDBObject();
322
		readStore.put("id", mdId);
323
		readStore.put("lastRead", new Date());
324
		values.add(readStore);
325
	}
326

  
327
	/**
328
	 * Gets the manager table.
329
	 *
330
	 * @return the managerTable
331
	 */
332
	public MongoCollection<DBObject> getManagerTable() {
333
		return managerTable;
334
	}
335

  
336
	/**
337
	 * Sets the manager table.
338
	 *
339
	 * @param managerTable
340
	 *            the managerTable to set
341
	 */
342
	public void setManagerTable(final MongoCollection<DBObject> managerTable) {
343
		this.managerTable = managerTable;
344
	}
345

  
346
	/*
347
	 * (non-Javadoc)
348
	 *
349
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#getInfoForCurrentMdStore(java.lang.String)
350
	 */
351
	@Override
352
	public MDStoreManagerInfo getInfoForCurrentMdStore(final String mdStoreId) throws MDStoreServiceException {
353
		verifyConsistency();
354
		final DBObject mdstoreInfo = getMDStoreAsDBObject(mdStoreId);
355
		if (mdstoreInfo == null) throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdStoreId);
356
		final MDStoreManagerInfo result = new MDStoreManagerInfo();
357
		result.setCurrentId((String) mdstoreInfo.get("currentId"));
358
		result.setMdId((String) mdstoreInfo.get("mdId"));
359
		final BasicDBList values = (BasicDBList) mdstoreInfo.get("expiring");
360
		for (Object v : values) {
361
			final MDStoreExpiredInfo stillused = new MDStoreExpiredInfo();
362
			final DBObject value = (DBObject) v;
363
			stillused.setId((String) value.get("id"));
364
			stillused.setLastRead((Date) value.get("lastRead"));
365
			result.addExpiredItem(stillused);
366
		}
367
		final BasicDBList transactions = (BasicDBList) mdstoreInfo.get("transactions");
368
		if (transactions != null) {
369
			for (Object tx : transactions) {
370
				final MDStoreTransactionInfo transaction = new MDStoreTransactionInfo();
371
				final DBObject value = (DBObject) tx;
372
				final String transactionId = (String) value.get("id");
373
				transaction.setId(transactionId);
374
				transaction.setDate((Date) value.get("date"));
375
				transaction.setRefresh((Boolean) value.get("refresh"));
376
				transaction.setSize(db.getCollection(transactionId).count());
377
				result.addTransactionInfo(transaction);
378
			}
379
		}
380
		return result;
381
	}
382

  
383
	/*
384
	 * (non-Javadoc)
385
	 *
386
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#dropUsed(java.lang.String, java.lang.String)
387
	 */
388
	@Override
389
	public Boolean dropUsed(final String mdId, final String idToDrop) throws MDStoreServiceException {
390
		verifyConsistency();
391
		final DBObject mdstoreInfo = getMDStoreAsDBObject(mdId);
392
		if (mdstoreInfo == null) throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdId);
393
		return dropStore(mdstoreInfo, idToDrop, "expiring");
394
	}
395

  
396
	private boolean dropStore(DBObject mdstoreInfo, final String idToDrop, String transactionListName) throws MDStoreServiceException {
397
		final BasicDBList transactionList = (BasicDBList) mdstoreInfo.get(transactionListName);
398
		for (int i = 0; i < transactionList.size(); i++) {
399
			final DBObject value = (DBObject) transactionList.get(i);
400
			final String currentUsedId = (String) value.get("id");
401
			if (currentUsedId.equals(idToDrop)) {
402
				db.getCollection(idToDrop).drop();
403
				db.getCollection("discarded-" + idToDrop).drop();
404
				transactionList.remove(value);
405
				mdstoreInfo.put(transactionListName, transactionList);
406
				this.getManagerTable().findOneAndReplace(new BasicDBObject("_id", mdstoreInfo.get("_id")), mdstoreInfo);
407
				return true;
408
			}
409
		}
410
		throw new MDStoreServiceException("Error, unable to drop collection " + idToDrop);
411
	}
412

  
413
	/*
414
	 * (non-Javadoc)
415
	 *
416
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#garbage()
417
	 */
418
	@Override
419
	public void garbage() throws MDStoreServiceException {
420
		verifyConsistency();
421
		log.info("Start garbage collection of MdStore");
422
		final FindIterable<DBObject> it = this.managerTable.find();
423
		int totalDeleted = 0;
424
		for (DBObject currentObject :  it){
425
			if (log.isDebugEnabled()) {
426
				log.debug("start to check id: " + currentObject.get("currentId"));
427
			}
428
			garbageExpiring(currentObject, (String) currentObject.get("currentId"));
429
			garbageTransactions(currentObject, (String) currentObject.get("currentId"));
430
			this.getManagerTable().findOneAndReplace(new BasicDBObject("_id", currentObject.get("_id")), currentObject);
431
		}
432

  
433
		// DELETING Collection that are not in the metadataManager table
434
		MongoIterable<String> collections = this.db.listCollectionNames();
435
		for (String collection : collections) {
436
			if ((collection.length() > 30) && (!collection.contains("discarded-"))) {
437
				final DBObject item = getMetadataObjectForCollections(collection);
438

  
439
				if (shouldDelete(collection, item)) {
440
					if (log.isDebugEnabled()) {
441
						log.debug("delete collection: " + collection + " from mongo");
442
					}
443
					db.getCollection(collection).drop();
444
					db.getCollection("discarded-" + collection).drop();
445
					if (log.isDebugEnabled()) {
446
						log.debug("delete collection: discarded-" + collection + " from mongo");
447
					}
448
				}
449
			}
450
		}
451

  
452
		log.info("Complete garbage collection of MdStore, total store deleted: " + totalDeleted);
453
	}
454

  
455
	private DBObject getMetadataObjectForCollections(final String collectionName) {
456
		if (collectionName == null) return null;
457
		final String postfix = "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";
458
		final String tmp = collectionName.contains("discarded-") ? StringUtils.substringAfter(collectionName, "discarded-") : collectionName;
459
		final String collectionNameCleaned = StringUtils.substringBefore(tmp, "::") + postfix;
460

  
461
		//DBObject query = QueryBuilder.start("mdId").is(collectionNameCleaned).get();
462
		Bson query = Filters.eq("mdId", collectionNameCleaned);
463
		return this.managerTable.find(query).first();
464

  
465
	}
466

  
467
	private boolean shouldDelete(final String collectionName, final DBObject metadataManagerInstance) {
468
		log.debug("should delete instance "+metadataManagerInstance);
469
		if((metadataManagerInstance== null) || (metadataManagerInstance.get("currentId")== null)){
470
			log.debug("the instance has not currentID");
471
			return true;
472
		}
473
		String currentId = (String) metadataManagerInstance.get("currentId");
474
		if (collectionName.equals(currentId)) return false;
475
		BasicDBList expiringList = (BasicDBList) metadataManagerInstance.get("expiring");
476
		if (findInList(expiringList, collectionName, "id")) return false;
477
		BasicDBList transactionsList = (BasicDBList) metadataManagerInstance.get("transactions");
478
		return !findInList(transactionsList, collectionName, "id");
479
	}
480

  
481
	private boolean findInList(final BasicDBList list, final String object, final String tagname) {
482
		if (list == null) return false;
483
		for (Object o : list) {
484
			DBObject currentObject = (DBObject) o;
485
			final String id = (String) currentObject.get(tagname);
486
			if (id.equals(object)) return true;
487
		}
488
		return false;
489
	}
490

  
491
	/**
492
	 * Delete.
493
	 *
494
	 * @param list
495
	 *            the list
496
	 * @param toRemove
497
	 *            the to remove
498
	 */
499
	private void delete(final BasicDBList list, final List<DBObject> toRemove) {
500

  
501
		for (final DBObject obj : toRemove) {
502
			if (log.isDebugEnabled()) {
503
				log.debug("deleting " + obj);
504
			}
505
			list.remove(obj);
506
		}
507
	}
508

  
509
	/**
510
	 * Garbage transactions.
511
	 *
512
	 * @param currentObject
513
	 *            the current object
514
	 * @param currentId
515
	 *            the current id
516
	 */
517
	private void garbageTransactions(final DBObject currentObject, final String currentId) {
518
		if (log.isDebugEnabled()) {
519
			log.debug("Start garbage transactions ");
520
		}
521

  
522
		final BasicDBList expiring = (BasicDBList) currentObject.get("transactions");
523
		if ((expiring == null) || (expiring.size() <= getMaxTransactions())) return;
524

  
525
		List<DBObject> expiringList = Lists.newArrayList();
526

  
527
		for (Object o : expiring) {
528
			final DBObject cobj = (DBObject) o;
529
			if (cobj != null) {
530
				expiringList.add((DBObject) o);
531
			}
532

  
533
		}
534

  
535
		expiringList.sort(MDStoreUtils.getComparatorOnDate());
536

  
537
		List<DBObject> toRemove = Lists.newArrayList();
538
		int i = 0;
539

  
540
		// We should remove the k item less recent
541
		// where k = numberOftotalTransaction - maxNumberOfTransaction
542
		// k = numberOfItemToRemove
543

  
544
		while (((expiringList.size() - toRemove.size()) > getMaxTransactions()) || (i < expiringList.size())) {
545
			DBObject currentObj = expiringList.get(i++);
546
			String objectId = (String) currentObj.get("id");
547
			if (!objectId.equals(currentId)) {
548
				if (log.isDebugEnabled()) {
549
					log.debug("delete collection: " + objectId + " from mongo");
550
				}
551
				db.getCollection(objectId).drop();
552
				db.getCollection("discarded-" + objectId).drop();
553
				if (log.isDebugEnabled()) {
554
					log.debug("delete collection: discarded-" + objectId + " from mongo");
555
				}
556
				toRemove.add(currentObj);
557
			} else {
558
				if (log.isDebugEnabled()) {
559
					log.debug("Cannot remove transaction " + objectId + " because is the currentId: " + currentId);
560
				}
561
			}
562
		}
563

  
564
		delete(expiring, toRemove);
565
		log.info("Deleted " + toRemove.size() + " transactions, mdStore Id:" + currentObject.get("mdId"));
566
	}
567

  
568
	/**
569
	 * Garbage expiring.
570
	 *
571
	 * @param currentObject
572
	 *            the current object
573
	 * @param currentId
574
	 *            the current id
575
	 */
576
	private void garbageExpiring(final DBObject currentObject, final String currentId) {
577
		if (log.isDebugEnabled()) {
578
			log.debug("Start to search expiring mdstores for id: " + currentObject.get("mdId"));
579
		}
580
		final BasicDBList expiring = (BasicDBList) currentObject.get("expiring");
581
		final List<DBObject> toRemove = Lists.newArrayList();
582
		if (log.isDebugEnabled()) {
583
			if (expiring == null) {
584
				log.debug("expiring list is null");
585
			} else {
586
				log.debug("expiring list size is :" + expiring.size());
587
			}
588
		}
589
		if ((expiring == null) || (expiring.size() == 0)) {
590
			log.debug("Deleted  0  expired  collections, mdStore Id:" + currentObject.get("mdId"));
591
			return;
592
		}
593
		for (Object anExpiring : expiring) {
594
			final DBObject currentExpiredStore = (DBObject) anExpiring;
595
			final String currentUsedId = (String) currentExpiredStore.get("id");
596
			final Days d = getExpiringDays(currentExpiredStore, "lastRead");
597
			if (log.isDebugEnabled()) {
598
				log.debug("the store :" + currentId + " expired since " + d.getDays() + "days ");
599
			}
600
			// DELETE the collection where the last time they was read
601
			// is more than 3 days ago
602
			if (d.getDays() > getExpiredDays()) {
603
				if (!currentUsedId.equals(currentId)) {
604
					db.getCollection(currentUsedId).drop();
605
					db.getCollection("discarded-" + currentUsedId).drop();
606
					log.debug("deleted collection " + currentUsedId);
607
				}
608
				toRemove.add(currentExpiredStore);
609
			}
610
		}
611
		delete(expiring, toRemove);
612
		log.debug("Deleted expired " + toRemove.size() + "collections, mdStore Id:" + currentObject.get("mdId"));
613
	}
614

  
615
	/**
616
	 * Gets the expiring days.
617
	 *
618
	 * @param value
619
	 *            the value
620
	 * @param paramName
621
	 *            the param name
622
	 * @return the expiring days
623
	 */
624
	private Days getExpiringDays(final DBObject value, final String paramName) {
625
		final Date lastRead = (Date) value.get(paramName);
626
		final DateTime last = new DateTime(lastRead);
627
		final DateTime today = new DateTime();
628
		final Days d = Days.daysBetween(last, today);
629
		return d;
630
	}
631

  
632
	/**
633
	 * Gets the expired days.
634
	 *
635
	 * @return the expiredDays
636
	 */
637
	public int getExpiredDays() {
638
		if (this.expiredDays == 0) return 3;
639
		return expiredDays;
640
	}
641

  
642
	/**
643
	 * Sets the expired days.
644
	 *
645
	 * @param expiredDays
646
	 *            the expiredDays to set
647
	 */
648
	public void setExpiredDays(final int expiredDays) {
649
		this.expiredDays = expiredDays;
650
	}
651

  
652
	/**
653
	 * Update incremental.
654
	 *
655
	 * @param transactionId
656
	 *            the transaction id
657
	 * @param currentId
658
	 *            the current id
659
	 */
660
	private void updateIncremental(final String transactionId, final String currentId) {
661
		final MongoCollection<DBObject> transaction = db.getCollection(transactionId, DBObject.class);
662
		final MongoCollection<DBObject> mdstore = db.getCollection(currentId, DBObject.class);
663
		final FindIterable<DBObject> it = transaction.find().noCursorTimeout(true);
664
		for (DBObject currentObj : it) {
665

  
666
			BasicDBObject newObj = new BasicDBObject();
667

  
668
			final String id = (String) currentObj.get("id");
669
			final String body = (String) currentObj.get("body");
670
			newObj.put("id", id);
671
			newObj.put("body", body);
672
			if (StringUtils.isNotBlank(id)) {
673
				//setting to journaled write concern to be sure that when the write returns everything has been flushed to disk (https://docs.mongodb.org/manual/faq/developers/#when-does-mongodb-write-updates-to-disk)
674
				//the explicit fsync command can't be run anymore: 'Command failed with error 13: 'fsync may only be run against the admin database.'
675
				final MongoCollection<DBObject> mdstoreWrite = mdstore.withWriteConcern(WriteConcern.JOURNALED);
676
				mdstoreWrite.replaceOne(new BasicDBObject("id", id), newObj, new UpdateOptions().upsert(true));
677
			}
678
		}
679
	}
680

  
681
	/*
682
	 * (non-Javadoc)
683
	 *
684
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#dropTransaction(java.lang.String, java.lang.String)
685
	 */
686
	@Override
687
	public Boolean dropTransaction(final String mdId, final String idToDrop) throws MDStoreServiceException {
688
		verifyConsistency();
689
		final DBObject mdstoreInfo = getMDStoreAsDBObject(mdId);
690
		if (mdstoreInfo == null) throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdId);
691
		return dropStore(mdstoreInfo, idToDrop, "transactions");
692
	}
693

  
694
	public void garbageTransactionsOnStart() throws MDStoreServiceException {
695
		verifyConsistency();
696
		FindIterable<DBObject> it = this.managerTable.find();
697

  
698
		final List<String> currentMdStoreId = Lists.newArrayList();
699
		for (DBObject currentObject : it){
700
			currentMdStoreId.add((String) currentObject.get("currentId"));
701
			final BasicDBList transactions = (BasicDBList) currentObject.get("transactions");
702
			if ((transactions != null) && (transactions.size() > 0)) {
703
				for (Object tx : transactions) {
704
					final DBObject currentTransactions = (DBObject) tx;
705
					final String id = (String) currentTransactions.get("id");
706
					db.getCollection(id).drop();
707
					db.getCollection("discarded-" + id).drop();
708
					log.debug("deleted collection " + id);
709
				}
710
				currentObject.put("transactions", new BasicDBList());
711
				this.getManagerTable().findOneAndReplace(new BasicDBObject("_id", currentObject.get("_id")), currentObject);
712
			}
713
		}
714

  
715
		//DELETING ALL THE DISCARDED COLLECTION THAT DISCARDED COLLECTION OF THE CURRENT MDSTORE
716
		final ArrayList<String> collectionsNames = Lists.newArrayList(db.listCollectionNames());
717

  
718
		for (String item : collectionsNames) {
719
			if (item.startsWith("discarded-")) {
720
				final String currentCollection = StringUtils.substringAfter(item, "discarded-");
721
				if (!currentMdStoreId.contains(currentCollection)) {
722
					log.info("Deleting discarded collection :" + item);
723
					this.db.getCollection(item).drop();
724
				}
725
			}
726
		}
727
	}
728

  
729
	/**
730
	 * Gets the max transactions.
731
	 *
732
	 * @return the maxTransactions
733
	 */
734
	public int getMaxTransactions() {
735
		return maxTransactions;
736
	}
737

  
738
	/**
739
	 * Sets the max transactions.
740
	 *
741
	 * @param maxTransactions
742
	 *            the maxTransactions to set
743
	 */
744
	public void setMaxTransactions(final int maxTransactions) {
745
		this.maxTransactions = maxTransactions;
746
	}
747
}
modules/cnr-mongo-mdstore/tags/cnr-mongo-mdstore-6.0.1/src/main/java/eu/dnetlib/data/mdstore/modular/mongodb/MongoResultSetListener.java
1
package eu.dnetlib.data.mdstore.modular.mongodb;
2

  
3
import java.util.ArrayList;
4
import java.util.List;
5
import java.util.regex.Pattern;
6

  
7
import com.google.common.base.Function;
8
import com.google.common.collect.Iterables;
9
import com.google.common.collect.Lists;
10
import com.mongodb.BasicDBObject;
11
import com.mongodb.DBObject;
12
import com.mongodb.MongoClient;
13
import com.mongodb.client.FindIterable;
14
import com.mongodb.client.MongoCollection;
15
import com.mongodb.client.model.Sorts;
16
import eu.dnetlib.enabling.resultset.ResultSet;
17
import eu.dnetlib.enabling.resultset.ResultSetAware;
18
import eu.dnetlib.enabling.resultset.ResultSetListener;
19
import eu.dnetlib.miscutils.maps.ConcurrentSizedMap;
20
import org.apache.commons.logging.Log;
21
import org.apache.commons.logging.LogFactory;
22
import org.bson.BsonDocument;
23
import org.bson.conversions.Bson;
24

  
25
import static com.mongodb.client.model.Filters.*;
26

  
27
public class MongoResultSetListener implements ResultSetListener, ResultSetAware {
28

  
29
	private static final Log log = LogFactory.getLog(MongoResultSetListener.class);
30

  
31
	private ConcurrentSizedMap<Integer, String> lastKeys = new ConcurrentSizedMap<>();
32
	private Bson sortByIdAsc = Sorts.orderBy(Sorts.ascending("id"));
33

  
34
	private Function<DBObject, String> serializer;
35
	private MongoCollection<DBObject> collection;
36
	private Bson query;
37

  
38
	public MongoResultSetListener(final MongoCollection<DBObject> collection, final Long from, final Long until, final Pattern filter, final Function<DBObject, String> serializer) {
39
		this.collection = collection;
40
		this.serializer = serializer;
... This diff was truncated because it exceeds the maximum size that can be displayed.

Also available in: Unified diff