Project

General

Profile

1
package eu.dnetlib.data.mdstore.modular.mongodb;
2

    
3
import java.util.Collections;
4
import java.util.Date;
5
import java.util.List;
6
import java.util.Set;
7

    
8
import org.apache.commons.lang.StringUtils;
9
import org.apache.commons.logging.Log;
10
import org.apache.commons.logging.LogFactory;
11
import org.joda.time.DateTime;
12
import org.joda.time.Days;
13
import org.springframework.beans.factory.annotation.Required;
14

    
15
import com.google.common.collect.Lists;
16
import com.mongodb.BasicDBList;
17
import com.mongodb.BasicDBObject;
18
import com.mongodb.DB;
19
import com.mongodb.DBCollection;
20
import com.mongodb.DBCursor;
21
import com.mongodb.DBObject;
22
import com.mongodb.QueryBuilder;
23
import com.mongodb.WriteConcern;
24

    
25
import eu.dnetlib.data.mdstore.MDStoreServiceException;
26
import eu.dnetlib.data.mdstore.modular.connector.MDStore;
27
import eu.dnetlib.data.mdstore.modular.connector.MDStoreExpiredInfo;
28
import eu.dnetlib.data.mdstore.modular.connector.MDStoreManagerInfo;
29
import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionInfo;
30
import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;
31
import eu.dnetlib.data.mdstore.modular.mongodb.utils.MDStoreUtils;
32

    
33
/**
34
 * The Class MDStoreTransactionManager.
35
 */
36
public class MDStoreTransactionManagerImpl implements MDStoreTransactionManager {
37

    
38
	/** The Constant log. */
39
	private static final Log log = LogFactory.getLog(MDStoreTransactionManagerImpl.class);
40

    
41
	/**
42
	 * The table name.
43
	 */
44
	private static String TABLE_NAME = "metadataManager";
45

    
46
	/** The max number of concurrent transactions per mdstore. */
47
	private int maxTransactions = 1;
48

    
49
	/**
50
	 * The db.
51
	 */
52

    
53
	private DB db;
54

    
55
	/**
56
	 * The manager table.
57
	 */
58
	private DBCollection managerTable;
59

    
60
	/** The expired days. */
61
	private int expiredDays;
62

    
63
	/**
64
	 * Bootstrap manager.
65
	 */
66
	private void bootstrapManager() {
67
		log.debug("Bootstrap Manager start");
68
		final DBCollection metadataColl = db.getCollection("metadata");
69
		final DBCursor values = metadataColl.find();
70
		this.setManagerTable(db.getCollection(TABLE_NAME));
71
		while (values.hasNext()) {
72
			final DBObject object = values.next();
73
			final String id = (String) object.get("mdId");
74
			String newId = null;
75
			if (id.contains("_")) {
76

    
77
				newId = StringUtils.substringBefore(id, "_");
78
			}
79

    
80
			final BasicDBObject input = new BasicDBObject();
81
			input.put("mdId", id);
82
			input.put("currentId", newId);
83
			input.put("expiring", new String[] {});
84
			input.put("transactions", new String[] {});
85
			getManagerTable().insert(input);
86
			log.debug(String.format("Added %s to Metadata Manager data structure", id));
87

    
88
		}
89
		final BasicDBObject ensureIndex = new BasicDBObject();
90
		ensureIndex.put("mdId", 1);
91
		log.debug("Create index in MetadaManager ");
92
		this.getManagerTable().createIndex(ensureIndex);
93
	}
94

    
95
	/**
96
	 * Verify consistency.
97
	 *
98
	 * @throws MDStoreServiceException
99
	 *             the MD store service exception
100
	 */
101
	@Override
102
	public void verifyConsistency() throws MDStoreServiceException {
103
		if (this.getManagerTable() == null) {
104
			if (!db.collectionExists(TABLE_NAME)) {
105
				bootstrapManager();
106
				if (this.getManagerTable() == null) { throw new MDStoreServiceException("Something bad happen, unable to create managerTable"); }
107
			} else {
108
				this.setManagerTable(db.getCollection(TABLE_NAME));
109
			}
110
		}
111
	}
112

    
113
	/**
114
	 * {@inheritDoc}
115
	 *
116
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#createMDStore(java.lang.String)
117
	 */
118
	@Override
119
	public void createMDStore(final String mdId) throws MDStoreServiceException {
120
		log.debug("Creating new mdstore");
121
		verifyConsistency();
122
		String newId = mdId;
123
		if (mdId.contains("_")) {
124
			newId = StringUtils.substringBefore(mdId, "_");
125
		}
126
		final BasicDBObject instance = new BasicDBObject();
127
		instance.put("mdId", mdId);
128
		instance.put("currentId", newId);
129
		instance.put("expiring", new String[] {});
130
		getManagerTable().insert(instance);
131
		this.getManagerTable().save(instance);
132
	}
133

    
134
	/**
135
	 * {@inheritDoc}
136
	 *
137
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#dropMDStore(java.lang.String)
138
	 */
139
	@Override
140
	public void dropMDStore(final String mdId) throws MDStoreServiceException {
141
		verifyConsistency();
142
		log.debug("Droping MDStore: " + mdId);
143
		final BasicDBObject query = new BasicDBObject();
144
		query.put("mdId", mdId);
145
		final DBCursor cursor = this.getManagerTable().find(query);
146
		if (cursor.hasNext() == false) { return; }
147
		final DBObject item = cursor.next();
148
		garbage();
149
		final String collectionName = (String) item.get("currentId");
150
		this.db.getCollection(collectionName).drop();
151
		this.db.getCollection("discarded-" + collectionName).drop();
152
		this.getManagerTable().remove(query);
153

    
154
	}
155

    
156
	/**
157
	 * {@inheritDoc}
158
	 *
159
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#getMDStoreCollection(java.lang.String)
160
	 */
161
	@Override
162
	public String getMDStoreCollection(final String mdId) throws MDStoreServiceException {
163
		verifyConsistency();
164
		final BasicDBObject query = new BasicDBObject();
165
		query.put("mdId", mdId);
166
		final DBCursor cursor = this.getManagerTable().find(query);
167
		if (cursor.hasNext() == false) {
168
			log.error("Error, unable to find Mdstore with Id " + mdId);
169
			throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdId);
170
		}
171
		return (String) cursor.next().get("currentId");
172
	}
173

    
174
	/**
175
	 * {@inheritDoc}
176
	 *
177
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#startTransaction(java.lang.String, boolean)
178
	 */
179
	@Override
180
	public String startTransaction(final String mdId, final boolean refresh) throws MDStoreServiceException {
181
		verifyConsistency();
182
		log.info("Start transaction for metadata store " + mdId);
183
		final BasicDBObject query = new BasicDBObject();
184
		query.put("mdId", mdId);
185
		final DBCursor cursor = this.getManagerTable().find(query);
186
		if (cursor.hasNext() == false) { throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdId); }
187

    
188
		String idCreation = StringUtils.substringBefore(mdId, "_");
189

    
190
		idCreation = idCreation + "::" + System.currentTimeMillis();
191
		final DBObject object = cursor.next();
192

    
193
		BasicDBList values;
194
		if (object.containsField("transactions")) {
195
			values = (BasicDBList) object.get("transactions");
196
			if (values.size() > getMaxTransactions()) { throw new MDStoreServiceException("Cannot create more than " + getMaxTransactions()
197
					+ " transactions, found: " + values.size() + ", mdId:" + mdId); }
198
		} else {
199
			values = new BasicDBList();
200
			object.put("transactions", values);
201
		}
202
		final BasicDBObject transactionMetadata = new BasicDBObject();
203
		transactionMetadata.put("id", idCreation.toString());
204
		transactionMetadata.put("refresh", refresh);
205
		transactionMetadata.put("date", new Date());
206
		values.add(transactionMetadata);
207
		this.getManagerTable().save(object);
208
		return idCreation.toString();
209
	}
210

    
211
	/**
212
	 * {@inheritDoc}
213
	 *
214
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#commit(java.lang.String, java.lang.String,
215
	 *      eu.dnetlib.data.mdstore.modular.connector.MDStore)
216
	 */
217
	@Override
218
	public boolean commit(final String transactionId, final String mdstoreId, final MDStore current) throws MDStoreServiceException {
219
		verifyConsistency();
220
		final BasicDBObject query = new BasicDBObject();
221
		query.put("mdId", mdstoreId);
222
		final DBCursor cursor = this.getManagerTable().find(query);
223
		if (cursor.hasNext() == false) { throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdstoreId); }
224
		final DBObject object = cursor.next();
225
		final BasicDBList transactions = (BasicDBList) object.get("transactions");
226
		final DBObject transaction = findTransaction(transactions, transactionId);
227
		if (transaction == null) { throw new MDStoreServiceException("Error, unable to find transaction with Id " + transactionId); }
228
		final boolean refresh = (Boolean) transaction.get("refresh");
229
		transactions.remove(transaction);
230
		final String oldId = (String) object.get("currentId");
231
		if (refresh) {
232
			object.put("currentId", transactionId);
233
			final BasicDBList stillUsed = (BasicDBList) object.get("expiring");
234
			if (stillUsed.size() == 0) {
235
				db.getCollection(oldId).drop();
236
				db.getCollection("discarded-" + oldId).drop();
237
			}
238
			log.debug("Replaced collection ");
239
		} else {
240
			log.debug("commit incremental ");
241
			updateIncremental(transactionId, oldId);
242
			db.getCollection(transactionId).drop();
243
			db.getCollection("discarded-" + transactionId).drop();
244
		}
245
		this.managerTable.save(object);
246

    
247
		log.info("Committed transaction for metadata store " + mdstoreId);
248
		return true;
249
	}
250

    
251
	/**
252
	 * Find transaction.
253
	 *
254
	 * @param transactions
255
	 *            the transactions
256
	 * @param transactionId
257
	 *            the transaction id
258
	 * @return the DB object
259
	 */
260
	private DBObject findTransaction(final BasicDBList transactions, final String transactionId) {
261
		if (transactions.size() == 0) { return null; }
262
		for (int i = 0; i < transactions.size(); i++) {
263
			final BasicDBObject transaction = (BasicDBObject) transactions.get(i);
264
			final String id = (String) transaction.get("id");
265
			if (transactionId.equals(id)) { return transaction; }
266
		}
267
		return null;
268

    
269
	}
270

    
271
	/**
272
	 * Gets the db.
273
	 *
274
	 * @return the db
275
	 */
276
	public DB getDb() {
277
		return db;
278
	}
279

    
280
	/**
281
	 * Sets the db.
282
	 *
283
	 * @param db
284
	 *            the db to set
285
	 */
286
	@Required
287
	public void setDb(final DB db) {
288
		this.db = db;
289
	}
290

    
291
	/**
292
	 * {@inheritDoc}
293
	 *
294
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#readMdStore(java.lang.String)
295
	 */
296
	@Override
297
	public String readMdStore(final String mdStoreId) throws MDStoreServiceException {
298
		verifyConsistency();
299
		final BasicDBObject query = new BasicDBObject();
300
		query.put("mdId", mdStoreId);
301
		final DBCursor cursor = this.getManagerTable().find(query);
302
		if (cursor.hasNext() == false) { throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdStoreId); }
303
		final DBObject object = cursor.next();
304
		final String currentId = (String) object.get("currentId");
305
		final BasicDBList values = (BasicDBList) object.get("expiring");
306
		updateMdstoreUsed(values, currentId);
307
		this.getManagerTable().save(object);
308
		return currentId;
309

    
310
	}
311

    
312
	/**
313
	 * Update mdstore used.
314
	 *
315
	 * @param values
316
	 *            the values
317
	 * @param mdId
318
	 *            the md id
319
	 */
320
	private void updateMdstoreUsed(final BasicDBList values, final String mdId) {
321
		if (values.size() > 0) {
322
			for (int i = 0; i < values.size(); i++) {
323
				final DBObject obj = (DBObject) values.get(i);
324
				final String id = (String) obj.get("id");
325
				if (mdId.equals(id)) {
326
					obj.put("lastRead", new Date());
327
					return;
328
				}
329
			}
330
		}
331
		final BasicDBObject readStore = new BasicDBObject();
332
		readStore.put("id", mdId);
333
		readStore.put("lastRead", new Date());
334
		values.add(readStore);
335
	}
336

    
337
	/**
338
	 * Gets the manager table.
339
	 *
340
	 * @return the managerTable
341
	 */
342
	public DBCollection getManagerTable() {
343
		return managerTable;
344
	}
345

    
346
	/**
347
	 * Sets the manager table.
348
	 *
349
	 * @param managerTable
350
	 *            the managerTable to set
351
	 */
352
	public void setManagerTable(final DBCollection managerTable) {
353
		this.managerTable = managerTable;
354
	}
355

    
356
	/*
357
	 * (non-Javadoc)
358
	 *
359
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#getInfoForCurrentMdStore(java.lang.String)
360
	 */
361
	@Override
362
	public MDStoreManagerInfo getInfoForCurrentMdStore(final String mdStoreId) throws MDStoreServiceException {
363
		verifyConsistency();
364
		final BasicDBObject query = new BasicDBObject();
365
		query.put("mdId", mdStoreId);
366
		final DBCursor cursor = this.getManagerTable().find(query);
367
		if (cursor.hasNext() == false) { throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdStoreId); }
368
		final DBObject object = cursor.next();
369
		final MDStoreManagerInfo result = new MDStoreManagerInfo();
370
		result.setCurrentId((String) object.get("currentId"));
371
		result.setMdId((String) object.get("mdId"));
372
		final BasicDBList values = (BasicDBList) object.get("expiring");
373
		for (int i = 0; i < values.size(); i++) {
374
			final MDStoreExpiredInfo stillused = new MDStoreExpiredInfo();
375
			final DBObject value = (DBObject) values.get(i);
376
			stillused.setId((String) value.get("id"));
377
			stillused.setLastRead((Date) value.get("lastRead"));
378
			result.addExpiredItem(stillused);
379
		}
380
		final BasicDBList transactions = (BasicDBList) object.get("transactions");
381
		if (transactions != null) {
382
			for (int i = 0; i < transactions.size(); i++) {
383
				final MDStoreTransactionInfo transaction = new MDStoreTransactionInfo();
384
				final DBObject value = (DBObject) transactions.get(i);
385
				final String transactionId = (String) value.get("id");
386
				transaction.setId(transactionId);
387
				transaction.setDate((Date) value.get("date"));
388
				transaction.setRefresh((Boolean) value.get("refresh"));
389
				transaction.setSize(db.getCollection(transactionId).count());
390
				result.addTransactionInfo(transaction);
391
			}
392
		}
393
		return result;
394
	}
395

    
396
	/*
397
	 * (non-Javadoc)
398
	 *
399
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#dropUsed(java.lang.String, java.lang.String)
400
	 */
401
	@Override
402
	public Boolean dropUsed(final String mdId, final String idToDrop) throws MDStoreServiceException {
403
		verifyConsistency();
404
		final BasicDBObject query = new BasicDBObject();
405
		query.put("mdId", mdId);
406
		final DBCursor cursor = this.getManagerTable().find(query);
407
		if (cursor.hasNext() == false) { throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdId); }
408
		final DBObject object = cursor.next();
409
		final BasicDBList values = (BasicDBList) object.get("expiring");
410
		for (int i = 0; i < values.size(); i++) {
411
			final DBObject value = (DBObject) values.get(i);
412
			final String currentUsedId = (String) value.get("id");
413
			if (currentUsedId.equals(idToDrop)) {
414
				db.getCollection(idToDrop).drop();
415
				db.getCollection("discarded-" + idToDrop).drop();
416
				values.remove(value);
417
				this.managerTable.save(object);
418
				return true;
419
			}
420
		}
421
		throw new MDStoreServiceException("Error, unable to drop old collection " + idToDrop);
422
	}
423

    
424
	/*
425
	 * (non-Javadoc)
426
	 *
427
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#garbage()
428
	 */
429
	@Override
430
	public void garbage() throws MDStoreServiceException {
431
		verifyConsistency();
432
		log.info("Start garbage collection of MdStore");
433
		final DBCursor cursor = this.managerTable.find();
434
		int totalDeleted = 0;
435
		while (cursor.hasNext()) {
436
			final DBObject currentObject = cursor.next();
437
			if (log.isDebugEnabled()) {
438
				log.debug("start to check id: " + (String) currentObject.get("currentId"));
439
			}
440
			garbageExpiring(currentObject, (String) currentObject.get("currentId"));
441
			garbageTransactions(currentObject, (String) currentObject.get("currentId"));
442
			this.managerTable.save(currentObject);
443
		}
444

    
445
		// DELETING Collection that are not in the metadataManager table
446
		Set<String> collections = this.db.getCollectionNames();
447
		for (String collection : collections) {
448
			if (collection.length() > 30 && collection.contains("discarded-") == false) {
449
				final DBObject item = getMetadataObjectForCollections(collection);
450

    
451
				if (shouldDelete(collection, item)) {
452
					if (log.isDebugEnabled()) {
453
						log.debug("delete collection: " + collection + " from mongo");
454
					}
455
					db.getCollection(collection).drop();
456
					db.getCollection("discarded-" + collection).drop();
457
					if (log.isDebugEnabled()) {
458
						log.debug("delete collection: discarded-" + collection + " from mongo");
459
					}
460
				}
461
			}
462
		}
463

    
464
		log.info("Complete garbage collection of MdStore, total store deleted: " + totalDeleted);
465
	}
466

    
467
	private DBObject getMetadataObjectForCollections(final String collectionName) {
468
		if (collectionName == null) { return null; }
469
		final String postfix = "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";
470
		final String tmp = collectionName.contains("discarded-") == true ? StringUtils.substringAfter(collectionName, "discarded-") : collectionName;
471
		final String collectionNameCleaned = StringUtils.substringBefore(tmp, "::") + postfix;
472

    
473
		DBObject query = QueryBuilder.start("mdId").is(collectionNameCleaned).get();
474
		return this.managerTable.findOne(query);
475
	}
476

    
477
	private boolean shouldDelete(final String collectionName, final DBObject metadataManagerInstance) {
478
		String currentId = (String) metadataManagerInstance.get("currentId");
479
		if (collectionName.equals(currentId)) { return false; }
480
		BasicDBList expiringList = (BasicDBList) metadataManagerInstance.get("expiring");
481
		if (findInList(expiringList, collectionName, "id") == true) { return false; }
482
		BasicDBList transactionsList = (BasicDBList) metadataManagerInstance.get("transactions");
483
		if (findInList(transactionsList, collectionName, "id") == true) { return false; }
484
		return true;
485
	}
486

    
487
	private boolean findInList(final BasicDBList list, final String object, final String tagname) {
488
		if (list == null) { return false; }
489
		for (int i = 0; i < list.size(); i++) {
490
			DBObject currentObject = (DBObject) list.get(i);
491
			final String id = (String) currentObject.get(tagname);
492
			if (id.equals(object)) { return true; }
493
		}
494
		return false;
495
	}
496

    
497
	/**
498
	 * Delete.
499
	 *
500
	 * @param list
501
	 *            the list
502
	 * @param toRemove
503
	 *            the to remove
504
	 */
505
	private void delete(final BasicDBList list, final List<DBObject> toRemove) {
506

    
507
		for (final DBObject obj : toRemove) {
508
			if (log.isDebugEnabled()) {
509
				log.debug("deleting " + obj);
510
			}
511
			list.remove(obj);
512
		}
513
	}
514

    
515
	/**
516
	 * Garbage transactions.
517
	 *
518
	 * @param currentObject
519
	 *            the current object
520
	 * @param currentId
521
	 *            the current id
522
	 */
523
	private void garbageTransactions(final DBObject currentObject, final String currentId) {
524
		if (log.isDebugEnabled()) {
525
			log.debug("Start garbage transactions ");
526
		}
527

    
528
		final BasicDBList expiring = (BasicDBList) currentObject.get("transactions");
529
		if (expiring == null || expiring.size() <= getMaxTransactions()) { return; }
530

    
531
		List<DBObject> expiringList = Lists.newArrayList();
532

    
533
		for (int i = 0; i < expiring.size(); i++) {
534
			final DBObject cobj = (DBObject) expiring.get(i);
535
			if (cobj != null) {
536
				expiringList.add((DBObject) expiring.get(i));
537
			}
538

    
539
		}
540

    
541
		Collections.sort(expiringList, MDStoreUtils.getComparatorOnDate());
542

    
543
		List<DBObject> toRemove = Lists.newArrayList();
544
		int i = 0;
545

    
546
		// We should remove the k item less recent
547
		// where k = numberOftotalTransaction - maxNumberOfTransaction
548
		// k = numberOfItemToRemove
549

    
550
		while (expiringList.size() - toRemove.size() > getMaxTransactions() || i < expiringList.size()) {
551
			DBObject currentObj = expiringList.get(i++);
552
			String objectId = (String) currentObj.get("id");
553
			if (!objectId.equals(currentId)) {
554
				if (log.isDebugEnabled()) {
555
					log.debug("delete collection: " + objectId + " from mongo");
556
				}
557
				db.getCollection(objectId).drop();
558
				db.getCollection("discarded-" + objectId).drop();
559
				if (log.isDebugEnabled()) {
560
					log.debug("delete collection: discarded-" + objectId + " from mongo");
561
				}
562
				toRemove.add(currentObj);
563
			} else {
564
				if (log.isDebugEnabled()) {
565
					log.debug("Cannot remove transaction " + objectId + " because is the currentId: " + currentId);
566
				}
567
			}
568
		}
569

    
570
		delete(expiring, toRemove);
571
		log.info("Deleted " + toRemove.size() + " transactions, mdStore Id:" + currentObject.get("mdId"));
572
	}
573

    
574
	/**
575
	 * Garbage expiring.
576
	 *
577
	 * @param currentObject
578
	 *            the current object
579
	 * @param currentId
580
	 *            the current id
581
	 */
582
	private void garbageExpiring(final DBObject currentObject, final String currentId) {
583
		if (log.isDebugEnabled()) {
584
			log.debug("Start to search expiring mdstores for id: " + currentObject.get("mdId"));
585
		}
586
		final BasicDBList expiring = (BasicDBList) currentObject.get("expiring");
587
		final List<DBObject> toRemove = Lists.newArrayList();
588
		if (log.isDebugEnabled()) {
589
			if (expiring == null) {
590
				log.debug("expiring list is null");
591
			} else {
592
				log.debug("expiring list size is :" + expiring.size());
593
			}
594
		}
595
		if (expiring == null || expiring.size() == 0) {
596
			log.info("Deleted  0  expired  collections, mdStore Id:" + currentObject.get("mdId"));
597
			return;
598
		}
599
		for (int i = 0; i < expiring.size(); i++) {
600
			final DBObject currentExpiredStore = (DBObject) expiring.get(i);
601
			final String currentUsedId = (String) currentExpiredStore.get("id");
602
			final Days d = getExpiringDays(currentExpiredStore, "lastRead");
603
			if (log.isDebugEnabled()) {
604
				log.debug("the store :" + currentId + " expired since " + d.getDays() + "days ");
605
			}
606
			// DELETE the collection where the last time they was read
607
			// is more than 3 days ago
608
			if (d.getDays() > getExpiredDays()) {
609
				if (currentUsedId.equals(currentId) == false) {
610
					db.getCollection(currentUsedId).drop();
611
					db.getCollection("discarded-" + currentUsedId).drop();
612
					log.debug("deleted collection " + currentUsedId);
613
				}
614
				toRemove.add(currentExpiredStore);
615
			}
616
		}
617
		delete(expiring, toRemove);
618
		log.info("Deleted expired " + toRemove.size() + "collections, mdStore Id:" + currentObject.get("mdId"));
619
	}
620

    
621
	/**
622
	 * Gets the expiring days.
623
	 *
624
	 * @param value
625
	 *            the value
626
	 * @param paramName
627
	 *            the param name
628
	 * @return the expiring days
629
	 */
630
	private Days getExpiringDays(final DBObject value, final String paramName) {
631
		final Date lastRead = (Date) value.get(paramName);
632
		final DateTime last = new DateTime(lastRead);
633
		final DateTime today = new DateTime();
634
		final Days d = Days.daysBetween(last, today);
635
		return d;
636
	}
637

    
638
	/**
639
	 * Gets the expired days.
640
	 *
641
	 * @return the expiredDays
642
	 */
643
	public int getExpiredDays() {
644
		if (this.expiredDays == 0) { return 3; }
645
		return expiredDays;
646
	}
647

    
648
	/**
649
	 * Sets the expired days.
650
	 *
651
	 * @param expiredDays
652
	 *            the expiredDays to set
653
	 */
654
	public void setExpiredDays(final int expiredDays) {
655
		this.expiredDays = expiredDays;
656
	}
657

    
658
	/**
659
	 * Update incremental.
660
	 *
661
	 * @param transactionId
662
	 *            the transaction id
663
	 * @param currentId
664
	 *            the current id
665
	 */
666
	private void updateIncremental(final String transactionId, final String currentId) {
667
		final DBCollection transaction = db.getCollection(transactionId);
668
		final DBCollection mdstore = db.getCollection(currentId);
669
		final DBCursor cursor = transaction.find();
670
		if (!cursor.hasNext()) { return; }
671
		DBObject currentObj = cursor.next();
672
		while (cursor.hasNext()) {
673

    
674
			BasicDBObject newObj = new BasicDBObject();
675

    
676
			final String id = (String) currentObj.get("id");
677
			final String body = (String) currentObj.get("body");
678
			newObj.put("id", id);
679
			newObj.put("body", body);
680
			if (id != null && !id.isEmpty()) {
681
				mdstore.update(new BasicDBObject("id", id), newObj, true, false, WriteConcern.ACKNOWLEDGED);
682
			}
683
			currentObj = cursor.next();
684
		}
685
		mdstore.getDB().command("{fsync:1}");
686
	}
687

    
688
	/*
689
	 * (non-Javadoc)
690
	 *
691
	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#dropTransaction(java.lang.String, java.lang.String)
692
	 */
693
	@Override
694
	public Boolean dropTransaction(final String mdId, final String idToDrop) throws MDStoreServiceException {
695
		verifyConsistency();
696
		final BasicDBObject query = new BasicDBObject();
697
		query.put("mdId", mdId);
698
		final DBCursor cursor = this.getManagerTable().find(query);
699
		if (cursor.hasNext() == false) { throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdId); }
700
		final DBObject object = cursor.next();
701
		final BasicDBList values = (BasicDBList) object.get("transactions");
702
		for (int i = 0; i < values.size(); i++) {
703
			final DBObject value = (DBObject) values.get(i);
704
			final String currentUsedId = (String) value.get("id");
705
			if (currentUsedId.equals(idToDrop)) {
706
				db.getCollection(idToDrop).drop();
707
				db.getCollection("discarded-" + idToDrop).drop();
708
				values.remove(value);
709
				this.managerTable.save(object);
710
				return true;
711
			}
712
		}
713
		throw new MDStoreServiceException("Error, unable to drop old collection " + idToDrop);
714
	}
715

    
716
	/**
717
	 * Gets the max transactions.
718
	 *
719
	 * @return the maxTransactions
720
	 */
721
	public int getMaxTransactions() {
722
		return maxTransactions;
723
	}
724

    
725
	/**
726
	 * Sets the max transactions.
727
	 *
728
	 * @param maxTransactions
729
	 *            the maxTransactions to set
730
	 */
731
	public void setMaxTransactions(final int maxTransactions) {
732
		this.maxTransactions = maxTransactions;
733
	}
734
}
(2-2/5)