/ - Diff - D-Net - D-Net project tracking tool

     import com.mongodb.client.result.DeleteResult;
     import com.mongodb.client.result.UpdateResult;
     import eu.dnetlib.cql.CqlTranslator;
     import eu.dnetlib.cql.mongo.MongoCqlTranslator;
     import eu.dnetlib.oai.PublisherField;
     import eu.dnetlib.oai.PublisherStore;
     import eu.dnetlib.oai.RecordChangeDetector;
-...
     		final Object sentinel = new Object();
     		this.dropDiscarded(source);
     		final Date feedDate = new Date();
     		final Thread background = new Thread(new Runnable() {
     			@Override
     			public void run() {
     				// For fast feeding we want to use a collection with unack write concern
     				final MongoCollection<DBObject> unackCollection = MongoPublisherStore.this.collection.withWriteConcern(WriteConcern.UNACKNOWLEDGED);
     				while (true) {
     					try {
     						final Object record = queue.take();
     						if (record == sentinel) {
     							break;
+    						}
     						safeFeedRecord((String) record, source, feedDate, unackCollection);
     					} catch (final InterruptedException e) {
     						log.fatal("got exception in background thread", e);
     						throw new IllegalStateException(e);
     		final Thread background = new Thread(() -> {
     			// For fast feeding we want to use a collection with unack write concern
     			final MongoCollection<DBObject> unackCollection = MongoPublisherStore.this.collection.withWriteConcern(WriteConcern.UNACKNOWLEDGED);
     			while (true) {
     				try {
     					final Object record = queue.take();
     					if (record == sentinel) {
     						break;
+    					}
     					safeFeedRecord((String) record, source, feedDate, unackCollection);
     				} catch (final InterruptedException e) {
     					log.fatal("got exception in background thread", e);
     					throw new IllegalStateException(e);
+    				}
+    			}
     		});
-...
     	 */
     	private boolean feedRecord(final String record, final String source, final Date feedDate, final MongoCollection<DBObject> unackCollection) {
     		final PublisherRecordParser parser = new PublisherRecordParser(this.mongoFields);
     		final Multimap<String, String> recordProperties = parser.parseRecord(record);
     		log.debug("configured parser for fields: "+this.mongoFields);
     		final Multimap<String, String> recordProperties = parser.parseRecord(record, source);
     		String id = "";
     		String oaiID = "";
     		if (recordProperties.containsKey(OAIConfigurationReader.ID_FIELD)) {
-...
+    			}
     		} else {
     			log.error("parsed record seems invalid -- no identifier property with name: " + OAIConfigurationReader.ID_FIELD);
     			log.debug("Extracted property map: \n"+recordProperties);
     			log.debug("from: \n"+record);
     			this.discardedCollection
     					.insertOne(new BasicDBObject(OAIConfigurationReader.SET_FIELD, source).append(OAIConfigurationReader.BODY_FIELD, record).append(
     							OAIConfigurationReader.DATESTAMP_FIELD, feedDate));

     		final Element elem = this.mongoOaistoreCache.get(k);
     		if (elem != null) {
     			log.debug("Store retreived from cache and alwaysNewRecord is" + this.alwaysNewRecord);
     			log.debug("Store retrieved from cache and alwaysNewRecord is" + this.alwaysNewRecord);
     			final MongoPublisherStore store = (MongoPublisherStore) elem.getObjectValue();
     			store.setAlwaysNewRecord(this.alwaysNewRecord);
     			log.debug(store);
     			return store;
     		} else {
     			log.debug("Store retreived, cache miss,  alwaysNewRecord is" + this.alwaysNewRecord);
     			log.debug("Store retrieved, cache miss,  alwaysNewRecord is" + this.alwaysNewRecord);
     			log.fatal("Not using cache to create oaistore from dbObject: " + k);
     			final MongoPublisherStore store = new MongoPublisherStore(storeId, mdFormat, mdInterpreation, mdLayout, db.getCollection(storeId, DBObject.class),
     					this.configuration.getFields(mdFormat, mdInterpreation, mdLayout), this.cqlTranslator, this.recordInfoGenerator,

     import com.google.common.collect.Iterables;
     import com.google.common.collect.Multimap;
     import eu.dnetlib.oai.PublisherField;
     import eu.dnetlib.oai.conf.OAIConfigurationExistReader;
     import org.apache.commons.lang3.StringUtils;
     import org.apache.commons.logging.Log;
     import org.apache.commons.logging.LogFactory;
     import org.dom4j.Document;
-...
+    	 *
     	 * @param record
     	 *            the XML string to parse.
     	 * @param source
     	 *             String identifying the source of the record. Can be null.
     	 * @return a Multimap describing the values to be indexed for this record.
     	 */
     	@SuppressWarnings({ "unchecked", "rawtypes" })
     	public Multimap<String, String> parseRecord(final String record) {
     	public Multimap<String, String> parseRecord(final String record, final String source) {
     		Multimap<String, String> recordProps = ArrayListMultimap.create();
     		try {
     			Document doc = this.saxReader.read(new StringReader(record));
     			if(StringUtils.isNotBlank(source)) recordProps.put(OAIConfigurationExistReader.SET_FIELD, source);
     			for (PublisherField field : this.storeIndices) {
     				for (Entry<String, String> indexEntry : field.getSources().entries()) {
     					// each xpath can return a list of nodes or strings, depending on the xpath
-...
     							recordProps.putAll(field.getFieldName(), xPathResult);
     						} else {
     							if (containsNodes(xPathResult)) {
     								recordProps.putAll(field.getFieldName(), Iterables.transform(xPathResult, obj -> {
     									if (obj == null) return "";
     									Node node = (Node) obj;
-...
     			recordProps = null;
+    		}
     		return recordProps;
+    	}
     	@SuppressWarnings("rawtypes")
-...
+    	}
     	public PublisherRecordParser(final List<PublisherField> storeIndices) {
     		super();
     		this.storeIndices = storeIndices;
+    	}

modules/dnet-data-provision-services/trunk/src/main/java/eu/dnetlib/oai/sync/OAIStoreSynchronizer.java
35	35	MongoPublisherStore store = this.getStore(sourceMetadataFormat, dbName, alwaysNewRecord);
36	36	int count = store.feed(records, recordSource);
37	37
38
39	38	log.info("Content synchronized: store " + sourceMetadataFormat + " fed with " + count + " records");
40	39	executeCallback(callback);
41	40	} catch (Exception e) {

Project

General

Profile

D-Net

Revision 45479

Added by Alessia Bardi over 7 years ago