Project

General

Profile

« Previous | Next » 

Revision 52044

long, not string!

View differences:

modules/cnr-mongo-mdstore/trunk/src/main/java/eu/dnetlib/data/mdstore/modular/mongodb/MongoBulkWritesManager.java
17 17
import org.apache.commons.logging.Log;
18 18
import org.apache.commons.logging.LogFactory;
19 19

  
20
import static eu.dnetlib.data.mdstore.modular.MDStoreConstants.*;
21

  
20 22
public class MongoBulkWritesManager {
21 23

  
22 24
	private static final Log log = LogFactory.getLog(MongoBulkWritesManager.class);
......
74 76
		}
75 77

  
76 78
		log.debug("found props: " + recordProperties);
77
		if (recordProperties.containsKey("id")) {
79
		if (recordProperties.containsKey(ID)) {
78 80
			final DBObject obj = buildDBObject(record, recordProperties, indexRecordField);
79 81
			if (log.isDebugEnabled()) {
80 82
				log.debug("Saving object" + obj);
81 83
			}
82
			validBulkOperationList.add(new ReplaceOneModel(new BasicDBObject("id", obj.get("id")), obj, new UpdateOptions().upsert(true)));
84
			validBulkOperationList.add(new ReplaceOneModel(new BasicDBObject(ID, obj.get(ID)), obj, new UpdateOptions().upsert(true)));
83 85
			if (((validBulkOperationList.size() % bulkSize) == 0) && !validBulkOperationList.isEmpty()) {
84 86
				validCollection.bulkWrite(validBulkOperationList, writeOptions);
85 87
				validBulkOperationList.clear();
......
93 95
	}
94 96

  
95 97
	private void discardRecord(final String record) {
96
		discardedBulkOperationList.add(new InsertOneModel(new BasicDBObject("body", record)));
98
		discardedBulkOperationList.add(new InsertOneModel(new BasicDBObject(BODY, record)));
97 99

  
98 100
		if (((discardedBulkOperationList.size() % bulkSize) == 0) && !discardedBulkOperationList.isEmpty()) {
99 101
			discardedCollection.bulkWrite(discardedBulkOperationList, writeOptions);
......
119 121

  
120 122
	private DBObject buildDBObject(final String record, final Map<String, String> recordProperties, final Map<String, List<String>> indexFieldProperties) {
121 123
		final DBObject obj = new BasicDBObject();
122
		obj.put("id", recordProperties.get("id"));
123
		obj.put("originalId", recordProperties.get("originalId"));
124
		obj.put("body", record);
125
		obj.put("timestamp", recordProperties.get("timestamp"));
124
		obj.put(ID, recordProperties.get(ID));
125
		obj.put(ORIGINALID, recordProperties.get(ORIGINALID));
126
		obj.put(BODY, record);
127
		obj.put(TIMESTAMP, Long.valueOf(recordProperties.get(TIMESTAMP)));
126 128
		if (indexFieldProperties != null)
127 129
			obj.putAll(indexFieldProperties);
128 130
		return obj;
modules/cnr-modular-mdstore-service/trunk/src/main/java/eu/dnetlib/data/mdstore/modular/SimpleRecordParser.java
11 11
import org.apache.commons.logging.LogFactory;
12 12
import org.xml.sax.InputSource;
13 13

  
14
import static eu.dnetlib.data.mdstore.modular.MDStoreConstants.*;
15

  
14 16
/**
15 17
 * Terrible implementation of a record parser.
16 18
 * 
......
25 27
	@Override
26 28
	public Map<String, String> parseRecord(String record) {
27 29
		Map<String, String> props = new HashMap<String, String>();
28
		props.put("timestamp", String.valueOf(getTimestamp()));
30
		props.put(TIMESTAMP, String.valueOf(getTimestamp()));
29 31

  
30 32
		try {
31 33
//			DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
......
34 36
//			Document doc = builder.parse(new InputSource(new StringReader(record)));
35 37
			InputSource doc = new InputSource(new StringReader(record));
36 38
			
37
			props.put("id", xpath.evaluate("//*[local-name()='objIdentifier']", doc));
39
			props.put(ID, xpath.evaluate("//*[local-name()='objIdentifier']", doc));
38 40
			props.put("originalId", xpath.evaluate("//*[local-name()='efgEntity']/*/*[local-name()='identifier']", doc));
39 41
			
40 42
//			String date = xpath.evaluate("//*[local-name()='dateOfCollection'][1]", doc);
modules/cnr-modular-mdstore-service/trunk/src/main/java/eu/dnetlib/data/mdstore/modular/StreamingRecordParser.java
10 10
import javax.xml.stream.XMLStreamException;
11 11
import javax.xml.stream.XMLStreamReader;
12 12

  
13
import static eu.dnetlib.data.mdstore.modular.MDStoreConstants.*;
14

  
13 15
/**
14 16
 * This method outperforms SimpleRecordParser by a vast amount, especially since we are just getting stuff in the
15 17
 * header.
......
29 31
			XMLStreamReader parser = factory.createXMLStreamReader(new ByteArrayInputStream(record.getBytes()));
30 32

  
31 33
			HashMap<String, String> res = new HashMap<String, String>();
32
			res.put("timestamp", String.valueOf(getTimestamp()));
34
			res.put(TIMESTAMP, String.valueOf(getTimestamp()));
33 35

  
34 36
			Stack<String> elementStack = new Stack<String>();
35 37
			elementStack.push("/");
......
43 45
					final String localName = parser.getLocalName();
44 46
					elementStack.push(localName);
45 47

  
46
					if ("objIdentifier".equals(localName)) {
48
					if (OBJIDENTIFIER.equals(localName)) {
47 49
						parser.next();
48 50

  
49
						res.put("id", parser.getText().trim());
51
						res.put(ID, parser.getText().trim());
50 52

  
51 53
					} else if ("identifier".equals(localName) && "efgEntity".equals(grandParent(elementStack))) {
52 54
						if (!res.containsKey("originalId")) {
......
61 63
//						log.info("ZZZZZZ: found identifier not at right depth " + elementStack + " grand parent " + grandParent(elementStack));
62 64
					}
63 65

  
64
					if (res.containsKey("id") && res.containsKey("originalId"))
66
					if (res.containsKey(ID) && res.containsKey("originalId"))
65 67
						return res;
66 68
				}
67 69
			}
modules/cnr-modular-mdstore-service/trunk/src/main/java/eu/dnetlib/data/mdstore/modular/MDStoreConstants.java
1
package eu.dnetlib.data.mdstore.modular;
2

  
3
public class MDStoreConstants {
4

  
5
	public static final String ID =  "id";
6
	public static final String OBJIDENTIFIER =  "objIdentifier";
7
	public static final String TIMESTAMP = "timestamp";
8
	public static final String ORIGINALID = "originalId";
9
	public static final String BODY = "body";
10

  
11

  
12
}

Also available in: Unified diff