Revision 50676
Added by Claudio Atzori about 6 years ago
| MDStoreTransactionManagerImpl.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.data.mdstore.modular.mongodb; |
| 2 | 2 |
|
| 3 | 3 |
import java.util.ArrayList; |
| 4 |
import java.util.Collections; |
|
| 5 | 4 |
import java.util.Date; |
| 6 | 5 |
import java.util.List; |
| 7 | 6 |
|
| ... | ... | |
| 20 | 19 |
import eu.dnetlib.data.mdstore.MDStoreServiceException; |
| 21 | 20 |
import eu.dnetlib.data.mdstore.modular.connector.*; |
| 22 | 21 |
import eu.dnetlib.data.mdstore.modular.mongodb.utils.MDStoreUtils; |
| 23 |
import org.apache.commons.lang.StringUtils; |
|
| 22 |
import org.apache.commons.lang3.StringUtils;
|
|
| 24 | 23 |
import org.apache.commons.logging.Log; |
| 25 | 24 |
import org.apache.commons.logging.LogFactory; |
| 26 | 25 |
import org.bson.conversions.Bson; |
| ... | ... | |
| 91 | 90 |
/** |
| 92 | 91 |
* Gets the DBObject describing an mdstore. null if there is no mdstore with the given id. |
| 93 | 92 |
* |
| 94 |
* @param mdstoreID |
|
| 93 |
* @param mdstoreID the mdStore identifier
|
|
| 95 | 94 |
* @return DBObject or null |
| 96 | 95 |
*/ |
| 97 |
private DBObject getMDStoreAsDBObject(String mdstoreID) throws MDStoreServiceException {
|
|
| 96 |
private DBObject getMDStoreAsDBObject(final String mdstoreID) throws MDStoreServiceException {
|
|
| 98 | 97 |
final BasicDBObject query = new BasicDBObject(); |
| 99 | 98 |
query.put("mdId", mdstoreID);
|
| 100 | 99 |
final FindIterable<DBObject> it = this.getManagerTable().find(query); |
| 101 |
DBObject mdstoreInfo = it.first(); |
|
| 102 |
return mdstoreInfo; |
|
| 100 |
return it.first(); |
|
| 103 | 101 |
} |
| 104 | 102 |
|
| 105 | 103 |
/** |
| ... | ... | |
| 196 | 194 |
values = new BasicDBList(); |
| 197 | 195 |
} |
| 198 | 196 |
final BasicDBObject transactionMetadata = new BasicDBObject(); |
| 199 |
transactionMetadata.put("id", idCreation.toString());
|
|
| 197 |
transactionMetadata.put("id", idCreation);
|
|
| 200 | 198 |
transactionMetadata.put("refresh", refresh);
|
| 201 | 199 |
transactionMetadata.put("date", new Date());
|
| 202 | 200 |
values.add(transactionMetadata); |
| 203 | 201 |
mdstoreInfo.put("transactions", values);
|
| 204 | 202 |
this.getManagerTable().findOneAndReplace(new BasicDBObject("_id", mdstoreInfo.get("_id")), mdstoreInfo);
|
| 205 |
return idCreation.toString();
|
|
| 203 |
return idCreation; |
|
| 206 | 204 |
} |
| 207 | 205 |
|
| 208 | 206 |
/** |
| ... | ... | |
| 253 | 251 |
*/ |
| 254 | 252 |
private DBObject findTransaction(final BasicDBList transactions, final String transactionId) {
|
| 255 | 253 |
if (transactions.size() == 0) return null; |
| 256 |
for (int i = 0; i < transactions.size(); i++) {
|
|
| 257 |
final BasicDBObject transaction = (BasicDBObject) transactions.get(i);
|
|
| 254 |
for (Object tx : transactions) {
|
|
| 255 |
final BasicDBObject transaction = (BasicDBObject) tx;
|
|
| 258 | 256 |
final String id = (String) transaction.get("id");
|
| 259 | 257 |
if (transactionId.equals(id)) return transaction; |
| 260 | 258 |
} |
| ... | ... | |
| 311 | 309 |
*/ |
| 312 | 310 |
private void updateMdstoreUsed(final BasicDBList values, final String mdId) {
|
| 313 | 311 |
if (values.size() > 0) {
|
| 314 |
for (int i = 0; i < values.size(); i++) {
|
|
| 315 |
final DBObject obj = (DBObject) values.get(i);
|
|
| 312 |
for (Object value : values) {
|
|
| 313 |
final DBObject obj = (DBObject) value; |
|
| 316 | 314 |
final String id = (String) obj.get("id");
|
| 317 | 315 |
if (mdId.equals(id)) {
|
| 318 | 316 |
obj.put("lastRead", new Date());
|
| ... | ... | |
| 359 | 357 |
result.setCurrentId((String) mdstoreInfo.get("currentId"));
|
| 360 | 358 |
result.setMdId((String) mdstoreInfo.get("mdId"));
|
| 361 | 359 |
final BasicDBList values = (BasicDBList) mdstoreInfo.get("expiring");
|
| 362 |
for (int i = 0; i < values.size(); i++) {
|
|
| 360 |
for (Object v : values) {
|
|
| 363 | 361 |
final MDStoreExpiredInfo stillused = new MDStoreExpiredInfo(); |
| 364 |
final DBObject value = (DBObject) values.get(i);
|
|
| 362 |
final DBObject value = (DBObject) v; |
|
| 365 | 363 |
stillused.setId((String) value.get("id"));
|
| 366 | 364 |
stillused.setLastRead((Date) value.get("lastRead"));
|
| 367 | 365 |
result.addExpiredItem(stillused); |
| 368 | 366 |
} |
| 369 | 367 |
final BasicDBList transactions = (BasicDBList) mdstoreInfo.get("transactions");
|
| 370 | 368 |
if (transactions != null) {
|
| 371 |
for (int i = 0; i < transactions.size(); i++) {
|
|
| 369 |
for (Object tx : transactions) {
|
|
| 372 | 370 |
final MDStoreTransactionInfo transaction = new MDStoreTransactionInfo(); |
| 373 |
final DBObject value = (DBObject) transactions.get(i);
|
|
| 371 |
final DBObject value = (DBObject) tx;
|
|
| 374 | 372 |
final String transactionId = (String) value.get("id");
|
| 375 | 373 |
transaction.setId(transactionId); |
| 376 | 374 |
transaction.setDate((Date) value.get("date"));
|
| ... | ... | |
| 435 | 433 |
// DELETING Collection that are not in the metadataManager table |
| 436 | 434 |
MongoIterable<String> collections = this.db.listCollectionNames(); |
| 437 | 435 |
for (String collection : collections) {
|
| 438 |
if ((collection.length() > 30) && (collection.contains("discarded-") == false)) {
|
|
| 436 |
if ((collection.length() > 30) && (!collection.contains("discarded-"))) {
|
|
| 439 | 437 |
final DBObject item = getMetadataObjectForCollections(collection); |
| 440 | 438 |
|
| 441 | 439 |
if (shouldDelete(collection, item)) {
|
| ... | ... | |
| 457 | 455 |
private DBObject getMetadataObjectForCollections(final String collectionName) {
|
| 458 | 456 |
if (collectionName == null) return null; |
| 459 | 457 |
final String postfix = "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ=="; |
| 460 |
final String tmp = collectionName.contains("discarded-") == true ? StringUtils.substringAfter(collectionName, "discarded-") : collectionName;
|
|
| 458 |
final String tmp = collectionName.contains("discarded-") ? StringUtils.substringAfter(collectionName, "discarded-") : collectionName;
|
|
| 461 | 459 |
final String collectionNameCleaned = StringUtils.substringBefore(tmp, "::") + postfix; |
| 462 | 460 |
|
| 463 | 461 |
//DBObject query = QueryBuilder.start("mdId").is(collectionNameCleaned).get();
|
| ... | ... | |
| 475 | 473 |
String currentId = (String) metadataManagerInstance.get("currentId");
|
| 476 | 474 |
if (collectionName.equals(currentId)) return false; |
| 477 | 475 |
BasicDBList expiringList = (BasicDBList) metadataManagerInstance.get("expiring");
|
| 478 |
if (findInList(expiringList, collectionName, "id") == true) return false;
|
|
| 476 |
if (findInList(expiringList, collectionName, "id")) return false; |
|
| 479 | 477 |
BasicDBList transactionsList = (BasicDBList) metadataManagerInstance.get("transactions");
|
| 480 |
return findInList(transactionsList, collectionName, "id") != true;
|
|
| 478 |
return !findInList(transactionsList, collectionName, "id");
|
|
| 481 | 479 |
} |
| 482 | 480 |
|
| 483 | 481 |
private boolean findInList(final BasicDBList list, final String object, final String tagname) {
|
| 484 | 482 |
if (list == null) return false; |
| 485 |
for (int i = 0; i < list.size(); i++) {
|
|
| 486 |
DBObject currentObject = (DBObject) list.get(i);
|
|
| 483 |
for (Object o : list) {
|
|
| 484 |
DBObject currentObject = (DBObject) o;
|
|
| 487 | 485 |
final String id = (String) currentObject.get(tagname); |
| 488 | 486 |
if (id.equals(object)) return true; |
| 489 | 487 |
} |
| ... | ... | |
| 526 | 524 |
|
| 527 | 525 |
List<DBObject> expiringList = Lists.newArrayList(); |
| 528 | 526 |
|
| 529 |
for (int i = 0; i < expiring.size(); i++) {
|
|
| 530 |
final DBObject cobj = (DBObject) expiring.get(i);
|
|
| 527 |
for (Object o : expiring) {
|
|
| 528 |
final DBObject cobj = (DBObject) o;
|
|
| 531 | 529 |
if (cobj != null) {
|
| 532 |
expiringList.add((DBObject) expiring.get(i));
|
|
| 530 |
expiringList.add((DBObject) o);
|
|
| 533 | 531 |
} |
| 534 | 532 |
|
| 535 | 533 |
} |
| 536 | 534 |
|
| 537 |
Collections.sort(expiringList, MDStoreUtils.getComparatorOnDate());
|
|
| 535 |
expiringList.sort(MDStoreUtils.getComparatorOnDate());
|
|
| 538 | 536 |
|
| 539 | 537 |
List<DBObject> toRemove = Lists.newArrayList(); |
| 540 | 538 |
int i = 0; |
| ... | ... | |
| 592 | 590 |
log.debug("Deleted 0 expired collections, mdStore Id:" + currentObject.get("mdId"));
|
| 593 | 591 |
return; |
| 594 | 592 |
} |
| 595 |
for (int i = 0; i < expiring.size(); i++) {
|
|
| 596 |
final DBObject currentExpiredStore = (DBObject) expiring.get(i);
|
|
| 593 |
for (Object anExpiring : expiring) {
|
|
| 594 |
final DBObject currentExpiredStore = (DBObject) anExpiring;
|
|
| 597 | 595 |
final String currentUsedId = (String) currentExpiredStore.get("id");
|
| 598 | 596 |
final Days d = getExpiringDays(currentExpiredStore, "lastRead"); |
| 599 | 597 |
if (log.isDebugEnabled()) {
|
| ... | ... | |
| 602 | 600 |
// DELETE the collection where the last time they was read |
| 603 | 601 |
// is more than 3 days ago |
| 604 | 602 |
if (d.getDays() > getExpiredDays()) {
|
| 605 |
if (currentUsedId.equals(currentId) == false) {
|
|
| 603 |
if (!currentUsedId.equals(currentId)) {
|
|
| 606 | 604 |
db.getCollection(currentUsedId).drop(); |
| 607 | 605 |
db.getCollection("discarded-" + currentUsedId).drop();
|
| 608 | 606 |
log.debug("deleted collection " + currentUsedId);
|
| ... | ... | |
| 702 | 700 |
currentMdStoreId.add((String) currentObject.get("currentId"));
|
| 703 | 701 |
final BasicDBList transactions = (BasicDBList) currentObject.get("transactions");
|
| 704 | 702 |
if ((transactions != null) && (transactions.size() > 0)) {
|
| 705 |
for (int i = 0; i < transactions.size(); i++) {
|
|
| 706 |
final DBObject currentTransactions = (DBObject) transactions.get(i);
|
|
| 703 |
for (Object tx : transactions) {
|
|
| 704 |
final DBObject currentTransactions = (DBObject) tx;
|
|
| 707 | 705 |
final String id = (String) currentTransactions.get("id");
|
| 708 | 706 |
db.getCollection(id).drop(); |
| 709 | 707 |
db.getCollection("discarded-" + id).drop();
|
| ... | ... | |
| 726 | 724 |
} |
| 727 | 725 |
} |
| 728 | 726 |
} |
| 729 |
|
|
| 730 |
|
|
| 731 |
|
|
| 732 |
|
|
| 733 | 727 |
} |
| 734 | 728 |
|
| 735 | 729 |
/** |
Also available in: Unified diff
using commons-lang3, using lambdas