Revision 52044
Added by Claudio Atzori almost 6 years ago
modules/cnr-mongo-mdstore/trunk/src/main/java/eu/dnetlib/data/mdstore/modular/mongodb/MongoBulkWritesManager.java | ||
---|---|---|
17 | 17 |
import org.apache.commons.logging.Log; |
18 | 18 |
import org.apache.commons.logging.LogFactory; |
19 | 19 |
|
20 |
import static eu.dnetlib.data.mdstore.modular.MDStoreConstants.*; |
|
21 |
|
|
20 | 22 |
public class MongoBulkWritesManager { |
21 | 23 |
|
22 | 24 |
private static final Log log = LogFactory.getLog(MongoBulkWritesManager.class); |
... | ... | |
74 | 76 |
} |
75 | 77 |
|
76 | 78 |
log.debug("found props: " + recordProperties); |
77 |
if (recordProperties.containsKey("id")) {
|
|
79 |
if (recordProperties.containsKey(ID)) {
|
|
78 | 80 |
final DBObject obj = buildDBObject(record, recordProperties, indexRecordField); |
79 | 81 |
if (log.isDebugEnabled()) { |
80 | 82 |
log.debug("Saving object" + obj); |
81 | 83 |
} |
82 |
validBulkOperationList.add(new ReplaceOneModel(new BasicDBObject("id", obj.get("id")), obj, new UpdateOptions().upsert(true)));
|
|
84 |
validBulkOperationList.add(new ReplaceOneModel(new BasicDBObject(ID, obj.get(ID)), obj, new UpdateOptions().upsert(true)));
|
|
83 | 85 |
if (((validBulkOperationList.size() % bulkSize) == 0) && !validBulkOperationList.isEmpty()) { |
84 | 86 |
validCollection.bulkWrite(validBulkOperationList, writeOptions); |
85 | 87 |
validBulkOperationList.clear(); |
... | ... | |
93 | 95 |
} |
94 | 96 |
|
95 | 97 |
private void discardRecord(final String record) { |
96 |
discardedBulkOperationList.add(new InsertOneModel(new BasicDBObject("body", record)));
|
|
98 |
discardedBulkOperationList.add(new InsertOneModel(new BasicDBObject(BODY, record)));
|
|
97 | 99 |
|
98 | 100 |
if (((discardedBulkOperationList.size() % bulkSize) == 0) && !discardedBulkOperationList.isEmpty()) { |
99 | 101 |
discardedCollection.bulkWrite(discardedBulkOperationList, writeOptions); |
... | ... | |
119 | 121 |
|
120 | 122 |
private DBObject buildDBObject(final String record, final Map<String, String> recordProperties, final Map<String, List<String>> indexFieldProperties) { |
121 | 123 |
final DBObject obj = new BasicDBObject(); |
122 |
obj.put("id", recordProperties.get("id"));
|
|
123 |
obj.put("originalId", recordProperties.get("originalId"));
|
|
124 |
obj.put("body", record);
|
|
125 |
obj.put("timestamp", recordProperties.get("timestamp"));
|
|
124 |
obj.put(ID, recordProperties.get(ID));
|
|
125 |
obj.put(ORIGINALID, recordProperties.get(ORIGINALID));
|
|
126 |
obj.put(BODY, record);
|
|
127 |
obj.put(TIMESTAMP, Long.valueOf(recordProperties.get(TIMESTAMP)));
|
|
126 | 128 |
if (indexFieldProperties != null) |
127 | 129 |
obj.putAll(indexFieldProperties); |
128 | 130 |
return obj; |
modules/cnr-modular-mdstore-service/trunk/src/main/java/eu/dnetlib/data/mdstore/modular/SimpleRecordParser.java | ||
---|---|---|
11 | 11 |
import org.apache.commons.logging.LogFactory; |
12 | 12 |
import org.xml.sax.InputSource; |
13 | 13 |
|
14 |
import static eu.dnetlib.data.mdstore.modular.MDStoreConstants.*; |
|
15 |
|
|
14 | 16 |
/** |
15 | 17 |
* Terrible implementation of a record parser. |
16 | 18 |
* |
... | ... | |
25 | 27 |
@Override |
26 | 28 |
public Map<String, String> parseRecord(String record) { |
27 | 29 |
Map<String, String> props = new HashMap<String, String>(); |
28 |
props.put("timestamp", String.valueOf(getTimestamp()));
|
|
30 |
props.put(TIMESTAMP, String.valueOf(getTimestamp()));
|
|
29 | 31 |
|
30 | 32 |
try { |
31 | 33 |
// DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); |
... | ... | |
34 | 36 |
// Document doc = builder.parse(new InputSource(new StringReader(record))); |
35 | 37 |
InputSource doc = new InputSource(new StringReader(record)); |
36 | 38 |
|
37 |
props.put("id", xpath.evaluate("//*[local-name()='objIdentifier']", doc));
|
|
39 |
props.put(ID, xpath.evaluate("//*[local-name()='objIdentifier']", doc));
|
|
38 | 40 |
props.put("originalId", xpath.evaluate("//*[local-name()='efgEntity']/*/*[local-name()='identifier']", doc)); |
39 | 41 |
|
40 | 42 |
// String date = xpath.evaluate("//*[local-name()='dateOfCollection'][1]", doc); |
modules/cnr-modular-mdstore-service/trunk/src/main/java/eu/dnetlib/data/mdstore/modular/StreamingRecordParser.java | ||
---|---|---|
10 | 10 |
import javax.xml.stream.XMLStreamException; |
11 | 11 |
import javax.xml.stream.XMLStreamReader; |
12 | 12 |
|
13 |
import static eu.dnetlib.data.mdstore.modular.MDStoreConstants.*; |
|
14 |
|
|
13 | 15 |
/** |
14 | 16 |
* This method outperforms SimpleRecordParser by a vast amount, especially since we are just getting stuff in the |
15 | 17 |
* header. |
... | ... | |
29 | 31 |
XMLStreamReader parser = factory.createXMLStreamReader(new ByteArrayInputStream(record.getBytes())); |
30 | 32 |
|
31 | 33 |
HashMap<String, String> res = new HashMap<String, String>(); |
32 |
res.put("timestamp", String.valueOf(getTimestamp()));
|
|
34 |
res.put(TIMESTAMP, String.valueOf(getTimestamp()));
|
|
33 | 35 |
|
34 | 36 |
Stack<String> elementStack = new Stack<String>(); |
35 | 37 |
elementStack.push("/"); |
... | ... | |
43 | 45 |
final String localName = parser.getLocalName(); |
44 | 46 |
elementStack.push(localName); |
45 | 47 |
|
46 |
if ("objIdentifier".equals(localName)) {
|
|
48 |
if (OBJIDENTIFIER.equals(localName)) {
|
|
47 | 49 |
parser.next(); |
48 | 50 |
|
49 |
res.put("id", parser.getText().trim());
|
|
51 |
res.put(ID, parser.getText().trim());
|
|
50 | 52 |
|
51 | 53 |
} else if ("identifier".equals(localName) && "efgEntity".equals(grandParent(elementStack))) { |
52 | 54 |
if (!res.containsKey("originalId")) { |
... | ... | |
61 | 63 |
// log.info("ZZZZZZ: found identifier not at right depth " + elementStack + " grand parent " + grandParent(elementStack)); |
62 | 64 |
} |
63 | 65 |
|
64 |
if (res.containsKey("id") && res.containsKey("originalId"))
|
|
66 |
if (res.containsKey(ID) && res.containsKey("originalId"))
|
|
65 | 67 |
return res; |
66 | 68 |
} |
67 | 69 |
} |
modules/cnr-modular-mdstore-service/trunk/src/main/java/eu/dnetlib/data/mdstore/modular/MDStoreConstants.java | ||
---|---|---|
1 |
package eu.dnetlib.data.mdstore.modular; |
|
2 |
|
|
3 |
public class MDStoreConstants { |
|
4 |
|
|
5 |
public static final String ID = "id"; |
|
6 |
public static final String OBJIDENTIFIER = "objIdentifier"; |
|
7 |
public static final String TIMESTAMP = "timestamp"; |
|
8 |
public static final String ORIGINALID = "originalId"; |
|
9 |
public static final String BODY = "body"; |
|
10 |
|
|
11 |
|
|
12 |
} |
Also available in: Unified diff
long, not string!