Revision 55329
Added by Alessia Bardi about 4 years ago
modules/cnr-mongo-mdstore/trunk/src/test/java/eu/dnetlib/data/mdstore/modular/mongodb/MongoBulkWritesManagerTest.java | ||
---|---|---|
1 |
package eu.dnetlib.data.mdstore.modular.mongodb; |
|
2 |
|
|
3 |
import java.util.Map; |
|
4 |
|
|
5 |
import com.google.common.collect.Maps; |
|
6 |
import com.mongodb.DBObject; |
|
7 |
import com.mongodb.WriteConcern; |
|
8 |
import com.mongodb.client.MongoCollection; |
|
9 |
import org.junit.Before; |
|
10 |
import org.junit.Test; |
|
11 |
import org.junit.runner.RunWith; |
|
12 |
import org.mockito.Mock; |
|
13 |
import org.mockito.MockitoAnnotations; |
|
14 |
import org.mockito.junit.MockitoJUnitRunner; |
|
15 |
|
|
16 |
import static org.mockito.Mockito.when; |
|
17 |
|
|
18 |
/** |
|
19 |
* Created by Alessia Bardi on 2019-04-12. |
|
20 |
* |
|
21 |
* @author Alessia Bardi |
|
22 |
*/ |
|
23 |
@RunWith(MockitoJUnitRunner.class) |
|
24 |
public class MongoBulkWritesManagerTest { |
|
25 |
|
|
26 |
private MongoBulkWritesManager mng; |
|
27 |
|
|
28 |
@Mock |
|
29 |
MongoCollection<DBObject> coll; |
|
30 |
|
|
31 |
@Before |
|
32 |
public void setUp(){ |
|
33 |
MockitoAnnotations.initMocks(this); |
|
34 |
when(coll.withWriteConcern(WriteConcern.ACKNOWLEDGED)).thenReturn(coll); |
|
35 |
mng = new MongoBulkWritesManager(coll, coll, null, 10, null, false); |
|
36 |
} |
|
37 |
|
|
38 |
@Test |
|
39 |
public void buildDBObjectTest(){ |
|
40 |
Map<String, String> props = Maps.newHashMap(); |
|
41 |
props.put("timestamp", "1555078665140"); |
|
42 |
props.put("id", "od______4301::5af4702a60ddf0615fd1dfd6ded104df"); |
|
43 |
props.put("originalId", "x"); |
|
44 |
props.put("body","<body/>"); |
|
45 |
|
|
46 |
DBObject obj = mng.buildDBObject("<x/>", props, null); |
|
47 |
System.out.println(obj); |
|
48 |
} |
|
49 |
} |
modules/cnr-mongo-mdstore/trunk/src/main/java/eu/dnetlib/data/mdstore/modular/mongodb/MongoBulkWritesManager.java | ||
---|---|---|
2 | 2 |
|
3 | 3 |
import java.util.List; |
4 | 4 |
import java.util.Map; |
5 |
import javax.annotation.Nullable; |
|
5 | 6 |
|
7 |
import com.google.common.base.Predicate; |
|
6 | 8 |
import com.google.common.collect.Lists; |
9 |
import com.google.common.collect.Maps; |
|
7 | 10 |
import com.mongodb.BasicDBObject; |
8 | 11 |
import com.mongodb.DBObject; |
9 | 12 |
import com.mongodb.WriteConcern; |
... | ... | |
51 | 54 |
this.discardRecords = discardRecords; |
52 | 55 |
this.mdref = mdref; |
53 | 56 |
|
54 |
this.indexRecords = this.mdref != null;
|
|
57 |
this.indexRecords = (this.mdref != null && !this.mdref.isEmpty());
|
|
55 | 58 |
this.writeOptions = new BulkWriteOptions().ordered(false); |
56 | 59 |
} |
57 | 60 |
|
... | ... | |
59 | 62 |
Map<String, String> recordProperties = null; |
60 | 63 |
try { |
61 | 64 |
recordProperties = recordParser.parseRecord(record); |
65 |
|
|
62 | 66 |
} catch (Throwable e) { |
63 | 67 |
if (discardRecords) { |
64 | 68 |
log.debug("unhandled exception: " + e.getMessage()); |
... | ... | |
119 | 123 |
discardedCollection = getCollectionWithWriteConcern(discardedCollection, WriteConcern.ACKNOWLEDGED); |
120 | 124 |
} |
121 | 125 |
|
122 |
private DBObject buildDBObject(final String record, final Map<String, String> recordProperties, final Map<String, List<String>> indexFieldProperties) {
|
|
126 |
protected DBObject buildDBObject(final String record, final Map<String, String> recordProperties, final Map<String, List<String>> indexFieldProperties) {
|
|
123 | 127 |
final DBObject obj = new BasicDBObject(); |
124 | 128 |
obj.put(ID, recordProperties.get(ID)); |
125 | 129 |
obj.put(ORIGINALID, recordProperties.get(ORIGINALID)); |
126 | 130 |
obj.put(BODY, record); |
127 | 131 |
obj.put(TIMESTAMP, Long.valueOf(recordProperties.get(TIMESTAMP))); |
128 | 132 |
if (indexFieldProperties != null) |
129 |
obj.putAll(indexFieldProperties); |
|
133 |
obj.putAll(Maps.filterKeys(indexFieldProperties, new Predicate<String>() { |
|
134 |
//ensure we do not override the mandatory fields above with some unexpected value |
|
135 |
@Override |
|
136 |
public boolean apply(@Nullable final String s) { |
|
137 |
return !s.equalsIgnoreCase(ID) && !s.equalsIgnoreCase(ORIGINALID) && !s.equalsIgnoreCase(BODY) && !s.equalsIgnoreCase(TIMESTAMP); |
|
138 |
} |
|
139 |
})); |
|
130 | 140 |
return obj; |
131 | 141 |
} |
132 | 142 |
|
Also available in: Unified diff
trying to understand why datestamp disappears on mongo