Revision 45455
Added by Alessia Bardi about 7 years ago
| MongoPublisherStore.java | ||
|---|---|---|
| 31 | 31 |
import com.mongodb.client.model.UpdateOptions; |
| 32 | 32 |
import com.mongodb.client.result.DeleteResult; |
| 33 | 33 |
import com.mongodb.client.result.UpdateResult; |
| 34 |
import eu.dnetlib.cql.CqlTranslator; |
|
| 35 |
import eu.dnetlib.cql.mongo.MongoCqlTranslator; |
|
| 34 | 36 |
import eu.dnetlib.oai.PublisherField; |
| 35 | 37 |
import eu.dnetlib.oai.PublisherStore; |
| 36 | 38 |
import eu.dnetlib.oai.RecordChangeDetector; |
| 37 | 39 |
import eu.dnetlib.oai.conf.OAIConfigurationReader; |
| 38 | 40 |
import eu.dnetlib.oai.info.RecordInfo; |
| 39 | 41 |
import eu.dnetlib.oai.info.SetInfo; |
| 40 |
import eu.dnetlib.oai.parser.MongoQueryParser; |
|
| 41 | 42 |
import eu.dnetlib.oai.parser.PublisherRecordParser; |
| 42 | 43 |
import eu.dnetlib.oai.sets.MongoSetCollection; |
| 43 | 44 |
import eu.dnetlib.rmi.provision.OaiPublisherRuntimeException; |
| ... | ... | |
| 60 | 61 |
private MongoCollection<DBObject> collection; |
| 61 | 62 |
private MongoCollection<DBObject> discardedCollection; |
| 62 | 63 |
|
| 64 |
private CqlTranslator cqlTranslator; |
|
| 63 | 65 |
private RecordInfoGenerator recordInfoGenerator; |
| 64 | 66 |
private MetadataExtractor metadataExtractor; |
| 65 | 67 |
|
| 66 |
private MongoQueryParser queryParser; |
|
| 67 |
|
|
| 68 | 68 |
private RecordChangeDetector recordChangeDetector; |
| 69 | 69 |
|
| 70 | 70 |
private MongoSetCollection mongoSetCollection; |
| ... | ... | |
| 93 | 93 |
final String layout, |
| 94 | 94 |
final MongoCollection<DBObject> collection, |
| 95 | 95 |
final List<PublisherField> mongoFields, |
| 96 |
final MongoQueryParser queryParser,
|
|
| 96 |
final CqlTranslator cqlTranslator,
|
|
| 97 | 97 |
final RecordInfoGenerator recordInfoGenerator, |
| 98 | 98 |
final String idScheme, |
| 99 | 99 |
final String idNamespace, |
| ... | ... | |
| 109 | 109 |
this.collection = collection; |
| 110 | 110 |
this.discardedCollection = mongodb.getCollection("discarded-" + collection.getNamespace().getCollectionName(), DBObject.class);
|
| 111 | 111 |
this.mongoFields = mongoFields; |
| 112 |
this.queryParser = queryParser;
|
|
| 112 |
this.cqlTranslator = cqlTranslator;
|
|
| 113 | 113 |
this.recordInfoGenerator = recordInfoGenerator; |
| 114 | 114 |
this.idScheme = idScheme; |
| 115 | 115 |
this.idNamespace = idNamespace; |
| 116 |
this.metadataExtractor = metadataExtractor; |
|
| 116 | 117 |
this.recordChangeDetector = recordChangeDetector; |
| 117 | 118 |
this.alwaysNewRecord = alwaysNewRecord; |
| 118 | 119 |
} |
| ... | ... | |
| 151 | 152 |
} |
| 152 | 153 |
|
| 153 | 154 |
private FindIterable<DBObject> loggedFindByQuery(final String queryString, final int limit) {
|
| 154 |
final Bson query = this.queryParser.parse(queryString);
|
|
| 155 |
final long start = System.currentTimeMillis();
|
|
| 156 |
final Bson sortByIdAsc = Sorts.orderBy(Sorts.ascending("_id"));
|
|
| 157 |
final FindIterable<DBObject> iter = this.collection.find(query).sort(sortByIdAsc).limit(limit);
|
|
| 158 |
final long end = System.currentTimeMillis();
|
|
| 155 |
final Bson query = parseQuery(queryString);
|
|
| 156 |
long start = System.currentTimeMillis(); |
|
| 157 |
Bson sortByIdAsc = Sorts.orderBy(Sorts.ascending("_id"));
|
|
| 158 |
FindIterable<DBObject> iter = this.collection.find(query).sort(sortByIdAsc).limit(limit); |
|
| 159 |
long end = System.currentTimeMillis(); |
|
| 159 | 160 |
log.debug("Query:" + query + "\ntime to get mongo iterable (ms): " + (end - start));
|
| 160 | 161 |
return iter; |
| 161 | 162 |
} |
| 162 | 163 |
|
| 164 |
private Bson parseQuery(final String query) {
|
|
| 165 |
try {
|
|
| 166 |
return cqlTranslator.toMongo(query); |
|
| 167 |
} catch(Exception e ) {
|
|
| 168 |
throw new OaiPublisherRuntimeException(e); |
|
| 169 |
} |
|
| 170 |
} |
|
| 171 |
|
|
| 172 |
|
|
| 163 | 173 |
@Override |
| 164 | 174 |
public List<PublisherField> getIndices() {
|
| 165 | 175 |
return this.mongoFields; |
| ... | ... | |
| 181 | 191 |
*/ |
| 182 | 192 |
@Override |
| 183 | 193 |
public void ensureIndices() {
|
| 194 |
final IndexOptions indexOptions = new IndexOptions().background(true); |
|
| 184 | 195 |
final Stopwatch sw = Stopwatch.createUnstarted(); |
| 185 | 196 |
sw.start(); |
| 186 |
for (final PublisherField field : this.mongoFields) {
|
|
| 187 |
createIndex(Lists.newArrayList(field.getFieldName())); |
|
| 197 |
for (PublisherField field : this.mongoFields) {
|
|
| 198 |
BasicDBObject mongoIdx = new BasicDBObject(field.getFieldName(), 1); |
|
| 199 |
log.debug("Creating index on store "+id+" : " + mongoIdx);
|
|
| 200 |
this.collection.createIndex(mongoIdx, indexOptions); |
|
| 188 | 201 |
} |
| 189 |
createIndex(Lists.newArrayList(OAIConfigurationReader.DATESTAMP_FIELD)); |
|
| 190 |
createIndex(Lists.newArrayList(OAIConfigurationReader.LAST_COLLECTION_DATE_FIELD)); |
|
| 202 |
log.debug("Creating index over : " + OAIConfigurationReader.DATESTAMP_FIELD);
|
|
| 203 |
this.collection.createIndex(new BasicDBObject(OAIConfigurationReader.DATESTAMP_FIELD, 1), indexOptions); |
|
| 204 |
log.debug("Creating index over : " + OAIConfigurationReader.LAST_COLLECTION_DATE_FIELD);
|
|
| 205 |
this.collection.createIndex(new BasicDBObject(OAIConfigurationReader.LAST_COLLECTION_DATE_FIELD, 1), indexOptions); |
|
| 191 | 206 |
sw.stop(); |
| 192 | 207 |
log.info("All indexes have been updated in " + sw.elapsed(TimeUnit.MILLISECONDS) + " milliseconds");
|
| 193 | 208 |
} |
| ... | ... | |
| 202 | 217 |
* List of fields to be included in the compound index |
| 203 | 218 |
* @theStore MongoPublisherStore where to create the index |
| 204 | 219 |
*/ |
| 205 |
public void createIndex(final List<String> fieldNames) {
|
|
| 220 |
public void createCompoundIndex(final List<String> fieldNames) {
|
|
| 206 | 221 |
if ((fieldNames == null) || fieldNames.isEmpty()) {
|
| 207 |
log.fatal("No fields specified for the creation of index");
|
|
| 222 |
log.fatal("No fields specified for the creation of the compound index");
|
|
| 208 | 223 |
} |
| 209 |
final BasicDBObjectBuilder theIndexBuilder = BasicDBObjectBuilder.start();
|
|
| 210 |
for (final String f : fieldNames) {
|
|
| 224 |
BasicDBObjectBuilder theIndexBuilder = BasicDBObjectBuilder.start(); |
|
| 225 |
for (String f : fieldNames) {
|
|
| 211 | 226 |
theIndexBuilder.add(f, 1); |
| 212 | 227 |
} |
| 213 |
final BasicDBObject theIndex = (BasicDBObject) theIndexBuilder.get();
|
|
| 228 |
BasicDBObject theIndex = (BasicDBObject) theIndexBuilder.get(); |
|
| 214 | 229 |
log.info("Creating index " + theIndex + " on " + this.getId());
|
| 215 | 230 |
this.getCollection().createIndex(theIndex, new IndexOptions().background(true)); |
| 216 | 231 |
} |
| ... | ... | |
| 323 | 338 |
|
| 324 | 339 |
@Override |
| 325 | 340 |
public void drop(final String queryString) {
|
| 326 |
final Bson query = this.queryParser.parse(queryString);
|
|
| 341 |
Bson query = parseQuery(queryString);
|
|
| 327 | 342 |
final DeleteResult deleteResult = this.collection.deleteMany(query); |
| 328 | 343 |
log.debug("Deleted by query: " + queryString + " #deleted: " + deleteResult.getDeletedCount());
|
| 329 |
|
|
| 330 | 344 |
} |
| 331 | 345 |
|
| 332 | 346 |
@Override |
| ... | ... | |
| 336 | 350 |
|
| 337 | 351 |
@Override |
| 338 | 352 |
public int count(final String queryString) {
|
| 339 |
if (StringUtils.isBlank(queryString)) { return (int) this.collection.count(); }
|
|
| 340 |
final Bson query = this.queryParser.parse(queryString);
|
|
| 353 |
if (StringUtils.isBlank(queryString)) return (int) this.collection.count();
|
|
| 354 |
Bson query = parseQuery(queryString);
|
|
| 341 | 355 |
return (int) this.collection.count(query); |
| 342 | 356 |
} |
| 343 | 357 |
|
| ... | ... | |
| 571 | 585 |
this.collection = collection; |
| 572 | 586 |
} |
| 573 | 587 |
|
| 574 |
public MongoQueryParser getQueryParser() {
|
|
| 575 |
return this.queryParser; |
|
| 576 |
} |
|
| 577 |
|
|
| 578 |
public void setQueryParser(final MongoQueryParser queryParser) {
|
|
| 579 |
this.queryParser = queryParser; |
|
| 580 |
} |
|
| 581 |
|
|
| 582 | 588 |
public MongoCollection<DBObject> getDiscardedCollection() {
|
| 583 | 589 |
return this.discardedCollection; |
| 584 | 590 |
} |
Also available in: Unified diff
Included changes about the CQL translation as in most recent version of cnr-cql-utils of DNet40 + refactoring of OAI parameters