Project

General

Profile

« Previous | Next » 

Revision 52044

Added by Claudio Atzori almost 6 years ago

long, not string!

View differences:

MongoBulkWritesManager.java
17 17 
import org.apache.commons.logging.Log;
18 18 
import org.apache.commons.logging.LogFactory;
19 19 

  		




  
  20
  
    
import static eu.dnetlib.data.mdstore.modular.MDStoreConstants.*;


  		




  
  21
  
    

  		




  20
  22
  
    
public class MongoBulkWritesManager {


  		




  21
  23
  
    

  		




  22
  24
  
    
	private static final Log log = LogFactory.getLog(MongoBulkWritesManager.class);


  		




  ......




  74
  76
  
    
		}


  		




  75
  77
  
    

  		




  76
  78
  
    
		log.debug("found props: " + recordProperties);


  		




  77
  
  
    
		if (recordProperties.containsKey("id")) {


  		




  
  79
  
    
		if (recordProperties.containsKey(ID)) {


  		




  78
  80
  
    
			final DBObject obj = buildDBObject(record, recordProperties, indexRecordField);


  		




  79
  81
  
    
			if (log.isDebugEnabled()) {


  		




  80
  82
  
    
				log.debug("Saving object" + obj);


  		




  81
  83
  
    
			}


  		




  82
  
  
    
			validBulkOperationList.add(new ReplaceOneModel(new BasicDBObject("id", obj.get("id")), obj, new UpdateOptions().upsert(true)));


  		




  
  84
  
    
			validBulkOperationList.add(new ReplaceOneModel(new BasicDBObject(ID, obj.get(ID)), obj, new UpdateOptions().upsert(true)));


  		




  83
  85
  
    
			if (((validBulkOperationList.size() % bulkSize) == 0) && !validBulkOperationList.isEmpty()) {


  		




  84
  86
  
    
				validCollection.bulkWrite(validBulkOperationList, writeOptions);


  		




  85
  87
  
    
				validBulkOperationList.clear();


  		




  ......




  93
  95
  
    
	}


  		




  94
  96
  
    

  		




  95
  97
  
    
	private void discardRecord(final String record) {


  		




  96
  
  
    
		discardedBulkOperationList.add(new InsertOneModel(new BasicDBObject("body", record)));


  		




  
  98
  
    
		discardedBulkOperationList.add(new InsertOneModel(new BasicDBObject(BODY, record)));


  		




  97
  99
  
    

  		




  98
  100
  
    
		if (((discardedBulkOperationList.size() % bulkSize) == 0) && !discardedBulkOperationList.isEmpty()) {


  		




  99
  101
  
    
			discardedCollection.bulkWrite(discardedBulkOperationList, writeOptions);


  		




  ......




  119
  121
  
    

  		




  120
  122
  
    
	private DBObject buildDBObject(final String record, final Map<String, String> recordProperties, final Map<String, List<String>> indexFieldProperties) {


  		




  121
  123
  
    
		final DBObject obj = new BasicDBObject();


  		




  122
  
  
    
		obj.put("id", recordProperties.get("id"));


  		




  123
  
  
    
		obj.put("originalId", recordProperties.get("originalId"));


  		




  124
  
  
    
		obj.put("body", record);


  		




  125
  
  
    
		obj.put("timestamp", recordProperties.get("timestamp"));


  		




  
  124
  
    
		obj.put(ID, recordProperties.get(ID));


  		




  
  125
  
    
		obj.put(ORIGINALID, recordProperties.get(ORIGINALID));


  		




  
  126
  
    
		obj.put(BODY, record);


  		




  
  127
  
    
		obj.put(TIMESTAMP, Long.valueOf(recordProperties.get(TIMESTAMP)));


  		




  126
  128
  
    
		if (indexFieldProperties != null)


  		




  127
  129
  
    
			obj.putAll(indexFieldProperties);


  		




  128
  130
  
    
		return obj;


  		












Also available in:  Unified diff