Revision 44144
Added by Alessia Bardi over 7 years ago
| MongoPublisherStore.java | ||
|---|---|---|
| 23 | 23 |
import com.mongodb.DBObject; |
| 24 | 24 |
import com.mongodb.WriteConcern; |
| 25 | 25 |
import com.mongodb.client.FindIterable; |
| 26 |
import com.mongodb.client.ListIndexesIterable; |
|
| 27 | 26 |
import com.mongodb.client.MongoCollection; |
| 28 | 27 |
import com.mongodb.client.MongoDatabase; |
| 29 | 28 |
import com.mongodb.client.model.Filters; |
| ... | ... | |
| 70 | 69 |
|
| 71 | 70 |
private MongoSetCollection mongoSetCollection; |
| 72 | 71 |
|
| 72 |
|
|
| 73 | 73 |
/** |
| 74 | 74 |
* Used to generate the OAI identifiers compliant to the protocol. See |
| 75 | 75 |
* http://www.openarchives.org/OAI/openarchivesprotocol.html#UniqueIdentifier. |
| ... | ... | |
| 172 | 172 |
* <p> |
| 173 | 173 |
* Note that by default ID_FIELD, SET_FIELD, DELETED_FIELD, BODY_FIELD, UPDATED_FIELD are not indexed. If you want an index on those, |
| 174 | 174 |
* then you have to specify it in the configuration file of the OAI Publisher: <br> |
| 175 |
* <INDEX name="deleted"> |
|
| 175 |
* <INDEX name="objIdentifier" repeatable="false"> |
|
| 176 |
* <SOURCE interpretation="transformed" layout="store" name="DMF" path="//*[local-name()='objIdentifier']"/> |
|
| 177 |
* </INDEX> |
|
| 176 | 178 |
* </p> |
| 177 | 179 |
* <p> |
| 178 | 180 |
* {@inheritDoc}
|
| 179 | 181 |
*/ |
| 180 | 182 |
@Override |
| 181 | 183 |
public void ensureIndices() {
|
| 182 |
final ListIndexesIterable<BasicDBObject> indexesIterable = this.collection.listIndexes(BasicDBObject.class); |
|
| 183 |
final IndexOptions indexOptions = new IndexOptions().background(true); |
|
| 184 | 184 |
final Stopwatch sw = Stopwatch.createUnstarted(); |
| 185 | 185 |
sw.start(); |
| 186 |
// I want to keep the composite indexes that might have been defined manually |
|
| 187 |
log.debug("Ensuring currently defined composite indexes:");
|
|
| 188 |
for (final BasicDBObject o : indexesIterable) {
|
|
| 189 |
final BasicDBObject fieldIndexed = (BasicDBObject) o.get("key");
|
|
| 190 |
if (fieldIndexed.keySet().size() > 1) {
|
|
| 191 |
log.debug(o); |
|
| 192 |
this.collection.createIndex(fieldIndexed, indexOptions); |
|
| 193 |
} |
|
| 194 |
} |
|
| 195 |
|
|
| 196 |
// Indexes on single fields. |
|
| 197 | 186 |
for (final PublisherField field : this.mongoFields) {
|
| 198 |
final BasicDBObject mongoIdx = new BasicDBObject(field.getFieldName(), 1); |
|
| 199 |
log.debug("Creating index : " + mongoIdx);
|
|
| 200 |
this.collection.createIndex(mongoIdx, indexOptions); |
|
| 187 |
createIndex(Lists.newArrayList(field.getFieldName())); |
|
| 201 | 188 |
} |
| 202 |
log.debug("Creating index over : " + OAIConfigurationReader.DATESTAMP_FIELD);
|
|
| 203 |
this.collection.createIndex(new BasicDBObject(OAIConfigurationReader.DATESTAMP_FIELD, 1), indexOptions); |
|
| 204 |
log.debug("Creating index over : " + OAIConfigurationReader.LAST_COLLECTION_DATE_FIELD);
|
|
| 205 |
this.collection.createIndex(new BasicDBObject(OAIConfigurationReader.LAST_COLLECTION_DATE_FIELD, 1), indexOptions); |
|
| 189 |
createIndex(Lists.newArrayList(OAIConfigurationReader.DATESTAMP_FIELD)); |
|
| 190 |
createIndex(Lists.newArrayList(OAIConfigurationReader.LAST_COLLECTION_DATE_FIELD)); |
|
| 206 | 191 |
sw.stop(); |
| 207 | 192 |
log.info("All indexes have been updated in " + sw.elapsed(TimeUnit.MILLISECONDS) + " milliseconds");
|
| 208 | 193 |
} |
| ... | ... | |
| 217 | 202 |
* List of fields to be included in the compound index |
| 218 | 203 |
* @theStore MongoPublisherStore where to create the index |
| 219 | 204 |
*/ |
| 220 |
public void createCompoundIndex(final List<String> fieldNames) {
|
|
| 205 |
public void createIndex(final List<String> fieldNames) {
|
|
| 221 | 206 |
if ((fieldNames == null) || fieldNames.isEmpty()) {
|
| 222 |
log.fatal("No fields specified for the creation of the compound index");
|
|
| 207 |
log.fatal("No fields specified for the creation of index");
|
|
| 223 | 208 |
} |
| 224 | 209 |
final BasicDBObjectBuilder theIndexBuilder = BasicDBObjectBuilder.start(); |
| 225 | 210 |
for (final String f : fieldNames) {
|
| ... | ... | |
| 283 | 268 |
final long endFeed = System.currentTimeMillis(); |
| 284 | 269 |
log.fatal("OAI STORE " + this.id + " FEEDING COMPLETED IN " + (endFeed - startFeed) + "ms");
|
| 285 | 270 |
this.setDeletedFlags(feedDate, source); |
| 271 |
//Let's add the set here so we can avoid to add in the workflow another job that upsert the sets. |
|
| 272 |
if(StringUtils.isNotBlank(source)) {
|
|
| 273 |
this.upsertSets(Lists.newArrayList(source)); |
|
| 274 |
} |
|
| 286 | 275 |
return this.count(); |
| 287 | 276 |
} |
| 288 | 277 |
|
Also available in: Unified diff
some cleanup in OAI related classes