/ - Diff - D-Net - D-Net project tracking tool

     import com.mongodb.client.ListIndexesIterable;
     import com.mongodb.client.MongoCollection;
     import com.mongodb.client.MongoCursor;
     import com.mongodb.client.model.IndexOptions;
     import eu.dnetlib.data.mdstore.MDStore;
     import eu.dnetlib.data.mdstore.model.MDStoreRecord;
     import eu.dnetlib.data.mdstore.model.ReadLock;
-...
     		final BlockingQueue<MDStoreRecord> queue = new ArrayBlockingQueue<>(BULK_SIZE);
     		final MDStoreRecord sentinel = new MDStoreRecord();
     		int countStored = 0;
     		final Callable<Integer> writer = () -> {
     			final MongoBulkWritesManager bulkWritesManager =
     		final Future<Integer> count = Executors.newSingleThreadExecutor().submit(() -> {
     			final MongoBulkWritesManager writer =
     					new MongoBulkWritesManager(recordCollection, discardedCollection, BULK_SIZE);
     			int count = 0;
     			int counter = 0;
     			while (true) {
     				try {
     					final MDStoreRecord MDStoreRecord = queue.take();
     					if (MDStoreRecord.equals(sentinel)) {
     						bulkWritesManager.flushBulks();
     					final MDStoreRecord record = queue.take();
     					if (record.equals(sentinel)) {
     						writer.flushBulks();
     						break;
+    					}
     					count++;
     					bulkWritesManager.insert(MDStoreRecord);
     					counter++;
     					writer.insert(record);
     				} catch (final InterruptedException e) {
     					log.fatal("got exception in background thread", e);
     					throw new IllegalStateException(e);
+    				}
+    			}
     			log.debug(String.format("extracted %s records from feeder queue", count));
     			return count;
     		};
     			log.debug(String.format("extracted %s records from feeder queue", counter));
     			return counter;
     		});
     		final Future<Integer> count = Executors.newSingleThreadExecutor().submit(writer);
     		try {
     			log.info("feeding mdstore " + id);
     			records.forEach(r -> {
-...
+    	}
     	public void ensureIndices() {
     		indexes.forEach(key -> recordCollection.createIndex(new BasicDBObject(key, 1)));
     		final IndexOptions op = new IndexOptions().background(true);
     		indexes.forEach(key -> recordCollection.createIndex(new BasicDBObject(key, 1), op));
+    	}
     	public boolean isIndexed() {

     						mongoDatabase.getCollection(name).drop();
     					});
     		//TODO check repositories has proper indexes
     		//createIndex(MDID, getTxManager());
     		log.info("Bootstrap mdstore complete!");
+    	}
-...
     		log.debug("Creating new mdstore");
     		final String mdId = getProfileManager().registerProfile(format, layout, interpretation);
     		final Metadata m = Metadata.create()
     		metadata.save(Metadata.create()
     				.setMdId(mdId)
     				.setCurrentId(mdId)
     				.setFormat(format)
     				.setLayout(layout)
     				.setInterpretation(interpretation);
     		metadata.save(m);
     				.setInterpretation(interpretation));
     		mongoDatabase.createCollection(mdId);
     		((MongoMDStore) getMDStore(mdId)).ensureIndices();
     		mongoDatabase.createCollection(m.getCurrentId());
     		createIndex(ID, mongoDatabase.getCollection(m.getCurrentId()));
     		log.info("Created new mdstore: " + mdId);
     		return mdId;
+    	}

     import com.google.common.collect.Lists;
     import com.google.common.collect.Maps;
     import org.springframework.data.mongodb.core.index.CompoundIndex;
     import org.springframework.data.mongodb.core.index.CompoundIndexes;
     import org.springframework.data.mongodb.core.index.Indexed;
     import org.springframework.data.mongodb.core.mapping.Document;
     @Document
     @CompoundIndexes({
     		@CompoundIndex(name = "fli_idx", def = "{'format': 1, 'layout': 1, 'interpretation': 1}")
     })
     public class Metadata {
     	@Indexed(unique = true)
-...
     	 * @return the expiring days
     	 */
     	private long getExpiringDays(final ReadLock info) {
     		final Date lastRead = info.getLastUpdateDate();
     		final LocalDate readDate = lastRead.toInstant().atZone(ZoneId.systemDefault()).toLocalDate();
     		final Long lastRead = info.getLastUpdate();
     		final LocalDate readDate = new Date(lastRead).toInstant().atZone(ZoneId.systemDefault()).toLocalDate();
     		return Duration.between(LocalDate.now().atTime(0, 0), readDate.atTime(0, 0)).toDays();
+    	}

     	private String id;
     	private int readCount;
     	private Date lastUpdateDate;
     	private Long lastUpdate;
     	public static ReadLock create(final String id) {
     		return new ReadLock()
     				.setId(id)
     				.setReadCount(0)
     				.setLastUpdateDate(new Date());
     				.setLastUpdate(new Date().getTime());
+    	}
     	public ReadLock incrementReadCount() {
-...
     	private ReadLock modifyReadCount(int v) {
     		setReadCount(getReadCount() + v);
     		setLastUpdateDate(new Date());
     		setLastUpdate(new Date().getTime());
     		return this;
+    	}
     	/**
     	 * @return the lastRead
     	 */
     	public Date getLastUpdateDate() {
     		return lastUpdateDate;
     	public Long getLastUpdate() {
     		return lastUpdate;
+    	}
     	/**
-...
     		return this;
+    	}
     	public ReadLock setLastUpdateDate(final Date lastUpdateDate) {
     		this.lastUpdateDate = lastUpdateDate;
     	public ReadLock setLastUpdate(final Long lastUpdate) {
     		this.lastUpdate = lastUpdate;
     		return this;
+    	}

     	@Field("id")
     	private String id;
     	private Boolean refresh;
     	private Date date;
     	private Long date;
     	public static Transaction create() {
     		return new Transaction()
     				.setDate(new Date());
     				.setDate(new Date().getTime());
+    	}
     	/**
     	 * @return the date
     	 */
     	public Date getDate() {
     	public Long getDate() {
     		return date;
+    	}
-...
     		return this;
+    	}
     	public Transaction setDate(final Date date) {
     	public Transaction setDate(final Long date) {
     		this.date = date;
     		return this;
+    	}

     package eu.dnetlib.data.mdstore.utils;
     import java.util.Comparator;
     import java.util.Date;
     import java.util.function.Function;
     import com.mongodb.DBObject;
     import eu.dnetlib.data.mdstore.model.Transaction;
     /**
      * Created by claudio on 13/03/2017.
      */
     public class MDStoreUtils {
     	public static Function<DBObject, String> mdId() {
     		return arg -> (String) arg.get("mdId");
+    	}
     	public static Comparator<Transaction> getComparatorOnDate() {
     		return (o1, o2) -> {
     			final Date d1 = o1.getDate();
     			final Date d2 = o2.getDate();
     			return d1.compareTo(d2);
     		};
+    	}
+    }

     					.setBody(s)
     					.setTimestamp(timestamp != null ? timestamp : new Date().getTime());
     			System.out.println(res.getTimestamp());
     			//System.out.println(res.getTimestamp());
     			while (parser.hasNext()) {
     				int event = parser.next();

Project

General

Profile

D-Net

Revision 46532

Added by Claudio Atzori about 7 years ago