Revision 42184
Added by Michele Artini about 8 years ago
| MongoPublisherStoreDAO.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.oai.mongo; |
| 2 | 2 |
|
| 3 | 3 |
import java.util.List; |
| 4 |
import java.util.stream.Collectors; |
|
| 5 |
|
|
| 4 | 6 |
import javax.annotation.Resource; |
| 5 | 7 |
|
| 6 |
import com.google.common.collect.Iterables; |
|
| 7 |
import com.google.common.collect.Lists; |
|
| 8 |
import eu.dnetlib.oai.PublisherStoreDAO; |
|
| 9 | 8 |
import org.apache.commons.logging.Log; |
| 10 | 9 |
import org.apache.commons.logging.LogFactory; |
| 11 | 10 |
import org.springframework.beans.factory.annotation.Autowired; |
| 12 | 11 |
import org.springframework.beans.factory.annotation.Required; |
| 13 | 12 |
|
| 13 |
import com.mongodb.BasicDBObjectBuilder; |
|
| 14 |
import com.mongodb.DBObject; |
|
| 15 |
import com.mongodb.MongoClient; |
|
| 16 |
import com.mongodb.client.FindIterable; |
|
| 17 |
import com.mongodb.client.MongoCollection; |
|
| 18 |
import com.mongodb.client.MongoDatabase; |
|
| 19 |
import com.mongodb.client.model.Filters; |
|
| 20 |
|
|
| 21 |
import eu.dnetlib.enabling.tools.DnetStreamSupport; |
|
| 22 |
import eu.dnetlib.oai.PublisherStoreDAO; |
|
| 23 |
import eu.dnetlib.oai.RecordChangeDetector; |
|
| 24 |
import eu.dnetlib.oai.conf.OAIConfigurationReader; |
|
| 25 |
import eu.dnetlib.oai.parser.MongoQueryParser; |
|
| 26 |
import eu.dnetlib.oai.sets.MongoSetCollection; |
|
| 27 |
import eu.dnetlib.rmi.provision.MDFInfo; |
|
| 28 |
import eu.dnetlib.rmi.provision.OaiPublisherException; |
|
| 29 |
import eu.dnetlib.rmi.provision.OaiPublisherRuntimeException; |
|
| 30 |
import net.sf.ehcache.Cache; |
|
| 31 |
import net.sf.ehcache.Element; |
|
| 32 |
|
|
| 14 | 33 |
public class MongoPublisherStoreDAO implements PublisherStoreDAO<MongoPublisherStore, DNetOAIMongoCursor> {
|
| 15 | 34 |
|
| 16 | 35 |
private static final Log log = LogFactory.getLog(MongoPublisherStoreDAO.class); // NOPMD by marko on 11/24/08 5:02 PM |
| ... | ... | |
| 39 | 58 |
/** |
| 40 | 59 |
* Cache for oaistores. Keys are dbName-storeId, values are objects of type MongoPublisherStore. |
| 41 | 60 |
*/ |
| 42 |
@Resource |
|
| 43 |
private EhCache<String, MongoPublisherStore> mongoOaistoreCache; |
|
| 61 |
private Cache mongoOaistoreCache; |
|
| 44 | 62 |
|
| 45 | 63 |
/** |
| 46 | 64 |
* Cache for oaistores. Keys is OAI metadata prefixes, values are objects of type MongoPublisherStore. |
| 47 | 65 |
*/ |
| 48 |
@Resource |
|
| 49 |
private EhCache<String, MongoPublisherStore> mongoOaistoreCacheByMdPrefix; |
|
| 66 |
private Cache mongoOaistoreCacheByMdPrefix; |
|
| 50 | 67 |
|
| 51 | 68 |
private boolean alwaysNewRecord; |
| 52 | 69 |
|
| ... | ... | |
| 58 | 75 |
public List<MongoPublisherStore> listPublisherStores(final String dbName) {
|
| 59 | 76 |
final MongoDatabase db = getDB(dbName); |
| 60 | 77 |
final FindIterable<DBObject> stores = db.getCollection(this.metadataCollection, DBObject.class).find(); |
| 61 |
return Lists.newArrayList( |
|
| 62 |
Iterables.transform(stores, new Function<DBObject, MongoPublisherStore>() {
|
|
| 63 |
@Override |
|
| 64 |
public MongoPublisherStore apply(final DBObject storeInfo) {
|
|
| 65 |
return createFromDBObject(storeInfo, db); |
|
| 66 |
} |
|
| 67 |
}) |
|
| 68 |
); |
|
| 78 |
return DnetStreamSupport.generateStreamFromIterator(stores.iterator()) |
|
| 79 |
.map(storeInfo -> createFromDBObject(storeInfo, db)) |
|
| 80 |
.collect(Collectors.toList()); |
|
| 69 | 81 |
} |
| 70 | 82 |
|
| 71 | 83 |
@Override |
| 72 | 84 |
public MongoPublisherStore getStore(final String storeId, final String dbName) {
|
| 73 |
DBObject storeInfo = getDB(dbName).getCollection(this.metadataCollection, DBObject.class).find(Filters.eq("id", storeId)).first();
|
|
| 85 |
final DBObject storeInfo = getDB(dbName).getCollection(this.metadataCollection, DBObject.class).find(Filters.eq("id", storeId)).first();
|
|
| 74 | 86 |
return this.createFromDBObject(storeInfo, getDB(dbName)); |
| 75 | 87 |
} |
| 76 | 88 |
|
| ... | ... | |
| 81 | 93 |
|
| 82 | 94 |
@Override |
| 83 | 95 |
public MongoPublisherStore getStoreFor(final String targetMetadataPrefix, final String dbName) {
|
| 84 |
if (mongoOaistoreCacheByMdPrefix.containsKey(targetMetadataPrefix)) return mongoOaistoreCacheByMdPrefix.get(targetMetadataPrefix); |
|
| 85 |
else {
|
|
| 86 |
MDFInfo info = this.configuration.getMetadataFormatInfo(targetMetadataPrefix); |
|
| 87 |
MongoPublisherStore store = this.getStore(info.getSourceFormatName(), info.getSourceFormatInterpretation(), info.getSourceFormatLayout(), dbName); |
|
| 88 |
mongoOaistoreCacheByMdPrefix.put(targetMetadataPrefix, store); |
|
| 96 |
|
|
| 97 |
final Element elem = this.mongoOaistoreCacheByMdPrefix.get(targetMetadataPrefix); |
|
| 98 |
|
|
| 99 |
if (elem != null) {
|
|
| 100 |
return (MongoPublisherStore) elem.getObjectValue(); |
|
| 101 |
} else {
|
|
| 102 |
final MDFInfo info = this.configuration.getMetadataFormatInfo(targetMetadataPrefix); |
|
| 103 |
final MongoPublisherStore store = |
|
| 104 |
this.getStore(info.getSourceFormatName(), info.getSourceFormatInterpretation(), info.getSourceFormatLayout(), dbName); |
|
| 105 |
this.mongoOaistoreCacheByMdPrefix.put(new Element(targetMetadataPrefix, store)); |
|
| 89 | 106 |
return store; |
| 90 | 107 |
} |
| 91 | 108 |
} |
| ... | ... | |
| 93 | 110 |
@Override |
| 94 | 111 |
public MongoPublisherStore createStore(final String mdfName, final String mdfInterpretation, final String mdfLayout, final String dbName) |
| 95 | 112 |
throws OaiPublisherException {
|
| 96 |
MongoDatabase db = getDB(dbName); |
|
| 97 |
DBObject store = createMetadataEntry(mdfName, mdfInterpretation, mdfLayout); |
|
| 98 |
MongoCollection<DBObject> metadata = db.getCollection(this.metadataCollection, DBObject.class); |
|
| 113 |
final MongoDatabase db = getDB(dbName);
|
|
| 114 |
final DBObject store = createMetadataEntry(mdfName, mdfInterpretation, mdfLayout);
|
|
| 115 |
final MongoCollection<DBObject> metadata = db.getCollection(this.metadataCollection, DBObject.class);
|
|
| 99 | 116 |
metadata.insertOne(store); |
| 100 |
MongoPublisherStore theStore = this.createFromDBObject(store, db); |
|
| 117 |
final MongoPublisherStore theStore = this.createFromDBObject(store, db);
|
|
| 101 | 118 |
return theStore; |
| 102 | 119 |
|
| 103 | 120 |
} |
| ... | ... | |
| 105 | 122 |
@Override |
| 106 | 123 |
public boolean deleteStore(final String storeId, final String dbName) {
|
| 107 | 124 |
|
| 108 |
MongoDatabase db = getDB(dbName); |
|
| 109 |
MongoCollection<DBObject> metadata = db.getCollection(this.metadataCollection, DBObject.class); |
|
| 125 |
final MongoDatabase db = getDB(dbName);
|
|
| 126 |
final MongoCollection<DBObject> metadata = db.getCollection(this.metadataCollection, DBObject.class);
|
|
| 110 | 127 |
final DBObject storeDeleted = metadata.findOneAndDelete(Filters.eq("id", storeId));
|
| 111 |
if (storeDeleted == null) return false; |
|
| 112 |
else {
|
|
| 128 |
if (storeDeleted == null) {
|
|
| 129 |
return false; |
|
| 130 |
} else {
|
|
| 113 | 131 |
db.getCollection(storeId).drop(); |
| 114 | 132 |
// TODO: should drop entries related to mdPrefix served by the store we are deleting, not all of them. |
| 115 | 133 |
this.mongoSetCollection.dropOAISets(dbName); |
| ... | ... | |
| 120 | 138 |
|
| 121 | 139 |
@Override |
| 122 | 140 |
public boolean deleteFromStore(final String storeId, final String dbName, final String set) {
|
| 123 |
MongoDatabase db = getDB(dbName); |
|
| 124 |
MongoCollection<DBObject> metadata = db.getCollection(this.metadataCollection, DBObject.class); |
|
| 125 |
DBObject storeInfo = metadata.find(Filters.eq("id", storeId)).first();
|
|
| 126 |
if (storeInfo == null) return false; |
|
| 127 |
else {
|
|
| 141 |
final MongoDatabase db = getDB(dbName); |
|
| 142 |
final MongoCollection<DBObject> metadata = db.getCollection(this.metadataCollection, DBObject.class); |
|
| 143 |
final DBObject storeInfo = metadata.find(Filters.eq("id", storeId)).first();
|
|
| 144 |
if (storeInfo == null) {
|
|
| 145 |
return false; |
|
| 146 |
} else {
|
|
| 128 | 147 |
db.getCollection(storeId).deleteOne(Filters.eq(OAIConfigurationReader.SET_FIELD, set)); |
| 129 | 148 |
this.mongoSetCollection.dropSet(dbName, set); |
| 130 | 149 |
log.debug("Deleted set " + set + " from oaistore " + storeId + ", db: " + dbName);
|
| ... | ... | |
| 143 | 162 |
} |
| 144 | 163 |
|
| 145 | 164 |
public void ensureIndex(final MongoPublisherStore store) {
|
| 146 |
if (store == null) throw new OaiPublisherRuntimeException("Can't ensure index on null store");
|
|
| 147 |
Thread t = new Thread() {
|
|
| 165 |
if (store == null) { throw new OaiPublisherRuntimeException("Can't ensure index on null store"); }
|
|
| 166 |
final Thread t = new Thread() {
|
|
| 148 | 167 |
|
| 149 | 168 |
@Override |
| 150 | 169 |
public void run() {
|
| ... | ... | |
| 155 | 174 |
} |
| 156 | 175 |
|
| 157 | 176 |
public void ensureIndex(final String dbName) {
|
| 158 |
List<MongoPublisherStore> stores = this.listPublisherStores(dbName); |
|
| 177 |
final List<MongoPublisherStore> stores = this.listPublisherStores(dbName);
|
|
| 159 | 178 |
for (final MongoPublisherStore s : stores) {
|
| 160 | 179 |
s.ensureIndices(); |
| 161 | 180 |
} |
| ... | ... | |
| 163 | 182 |
} |
| 164 | 183 |
|
| 165 | 184 |
private MongoPublisherStore createFromDBObject(final DBObject storeInfo, final MongoDatabase db) {
|
| 166 |
if (storeInfo == null) return null; |
|
| 167 |
String storeId = (String) storeInfo.get("id");
|
|
| 168 |
String mdFormat = (String) storeInfo.get("metadataFormat");
|
|
| 169 |
String mdInterpreation = (String) storeInfo.get("interpretation");
|
|
| 170 |
String mdLayout = (String) storeInfo.get("layout");
|
|
| 171 |
String k = db.getName() + "-" + storeId; |
|
| 172 |
if (mongoOaistoreCache.containsKey(k)) {
|
|
| 173 |
log.debug("Store retreived from cache and alwaysNewRecord is" + alwaysNewRecord);
|
|
| 174 |
MongoPublisherStore store = mongoOaistoreCache.get(k); |
|
| 175 |
store.setAlwaysNewRecord(alwaysNewRecord); |
|
| 185 |
if (storeInfo == null) { return null; }
|
|
| 186 |
final String storeId = (String) storeInfo.get("id");
|
|
| 187 |
final String mdFormat = (String) storeInfo.get("metadataFormat");
|
|
| 188 |
final String mdInterpreation = (String) storeInfo.get("interpretation");
|
|
| 189 |
final String mdLayout = (String) storeInfo.get("layout");
|
|
| 190 |
final String k = db.getName() + "-" + storeId; |
|
| 191 |
|
|
| 192 |
final Element elem = this.mongoOaistoreCache.get(k); |
|
| 193 |
if (elem != null) {
|
|
| 194 |
log.debug("Store retreived from cache and alwaysNewRecord is" + this.alwaysNewRecord);
|
|
| 195 |
final MongoPublisherStore store = (MongoPublisherStore) elem.getObjectValue(); |
|
| 196 |
store.setAlwaysNewRecord(this.alwaysNewRecord); |
|
| 176 | 197 |
return store; |
| 177 | 198 |
} else {
|
| 178 |
log.debug("Store retreived, cache miss, alwaysNewRecord is" + alwaysNewRecord);
|
|
| 199 |
log.debug("Store retreived, cache miss, alwaysNewRecord is" + this.alwaysNewRecord);
|
|
| 179 | 200 |
log.fatal("Not using cache to create oaistore from dbObject: " + k);
|
| 180 |
MongoPublisherStore store = new MongoPublisherStore(storeId, mdFormat, mdInterpreation, mdLayout, db.getCollection(storeId, DBObject.class), |
|
| 181 |
this.configuration.getFields(mdFormat, mdInterpreation, mdLayout), queryParser, recordInfoGenerator, this.configuration.getIdScheme(), |
|
| 182 |
this.configuration.getIdNamespace(), this.metadataExtractor, this.recordChangeDetector, alwaysNewRecord, db); |
|
| 183 |
store.setMongoSetCollection(mongoSetCollection); |
|
| 184 |
mongoOaistoreCache.put(k, store); |
|
| 201 |
final MongoPublisherStore store = new MongoPublisherStore(storeId, mdFormat, mdInterpreation, mdLayout, db.getCollection(storeId, DBObject.class), |
|
| 202 |
this.configuration.getFields(mdFormat, mdInterpreation, mdLayout), this.queryParser, this.recordInfoGenerator, |
|
| 203 |
this.configuration.getIdScheme(), |
|
| 204 |
this.configuration.getIdNamespace(), this.metadataExtractor, this.recordChangeDetector, this.alwaysNewRecord, db); |
|
| 205 |
store.setMongoSetCollection(this.mongoSetCollection); |
|
| 206 |
this.mongoOaistoreCache.put(new Element(k, store)); |
|
| 185 | 207 |
return store; |
| 186 | 208 |
} |
| 187 | 209 |
} |
| 188 | 210 |
|
| 189 | 211 |
private DBObject createMetadataEntry(final String mdfName, final String mdfInterpretation, final String mdfLayout) {
|
| 190 |
DBObject info = BasicDBObjectBuilder.start("id", generateStoreId(mdfName, mdfInterpretation, mdfLayout)).append("metadataFormat", mdfName)
|
|
| 212 |
final DBObject info = BasicDBObjectBuilder.start("id", generateStoreId(mdfName, mdfInterpretation, mdfLayout)).append("metadataFormat", mdfName)
|
|
| 191 | 213 |
.append("interpretation", mdfInterpretation).append("layout", mdfLayout).get();
|
| 192 | 214 |
return info; |
| 193 | 215 |
|
| ... | ... | |
| 198 | 220 |
} |
| 199 | 221 |
|
| 200 | 222 |
public String getMetadataCollection() {
|
| 201 |
return metadataCollection; |
|
| 223 |
return this.metadataCollection;
|
|
| 202 | 224 |
} |
| 203 | 225 |
|
| 204 | 226 |
@Required |
| ... | ... | |
| 207 | 229 |
} |
| 208 | 230 |
|
| 209 | 231 |
public MongoQueryParser getQueryParser() {
|
| 210 |
return queryParser; |
|
| 232 |
return this.queryParser;
|
|
| 211 | 233 |
} |
| 212 | 234 |
|
| 213 | 235 |
public void setQueryParser(final MongoQueryParser queryParser) {
|
| ... | ... | |
| 215 | 237 |
} |
| 216 | 238 |
|
| 217 | 239 |
public OAIConfigurationReader getConfiguration() {
|
| 218 |
return configuration; |
|
| 240 |
return this.configuration;
|
|
| 219 | 241 |
} |
| 220 | 242 |
|
| 221 | 243 |
public void setConfiguration(final OAIConfigurationReader configuration) {
|
| ... | ... | |
| 223 | 245 |
} |
| 224 | 246 |
|
| 225 | 247 |
public RecordInfoGenerator getRecordInfoGenerator() {
|
| 226 |
return recordInfoGenerator; |
|
| 248 |
return this.recordInfoGenerator;
|
|
| 227 | 249 |
} |
| 228 | 250 |
|
| 229 | 251 |
public void setRecordInfoGenerator(final RecordInfoGenerator recordInfoGenerator) {
|
| ... | ... | |
| 231 | 253 |
} |
| 232 | 254 |
|
| 233 | 255 |
public MetadataExtractor getMetadataExtractor() {
|
| 234 |
return metadataExtractor; |
|
| 256 |
return this.metadataExtractor;
|
|
| 235 | 257 |
} |
| 236 | 258 |
|
| 237 | 259 |
public void setMetadataExtractor(final MetadataExtractor metadataExtractor) {
|
| ... | ... | |
| 239 | 261 |
} |
| 240 | 262 |
|
| 241 | 263 |
public RecordChangeDetector getRecordChangeDetector() {
|
| 242 |
return recordChangeDetector; |
|
| 264 |
return this.recordChangeDetector;
|
|
| 243 | 265 |
} |
| 244 | 266 |
|
| 245 | 267 |
public void setRecordChangeDetector(final RecordChangeDetector recordChangeDetector) {
|
| ... | ... | |
| 247 | 269 |
} |
| 248 | 270 |
|
| 249 | 271 |
public MongoSetCollection getMongoSetCollection() {
|
| 250 |
return mongoSetCollection; |
|
| 272 |
return this.mongoSetCollection;
|
|
| 251 | 273 |
} |
| 252 | 274 |
|
| 253 | 275 |
public void setMongoSetCollection(final MongoSetCollection mongoSetCollection) {
|
| ... | ... | |
| 255 | 277 |
} |
| 256 | 278 |
|
| 257 | 279 |
public boolean isAlwaysNewRecord() {
|
| 258 |
return alwaysNewRecord; |
|
| 280 |
return this.alwaysNewRecord;
|
|
| 259 | 281 |
} |
| 260 | 282 |
|
| 261 | 283 |
public void setAlwaysNewRecord(final boolean alwaysNewRecord) {
|
| 262 | 284 |
this.alwaysNewRecord = alwaysNewRecord; |
| 263 | 285 |
} |
| 264 | 286 |
|
| 287 |
public Cache getMongoOaistoreCache() {
|
|
| 288 |
return this.mongoOaistoreCache; |
|
| 289 |
} |
|
| 290 |
|
|
| 291 |
@Required |
|
| 292 |
public void setMongoOaistoreCache(final Cache mongoOaistoreCache) {
|
|
| 293 |
this.mongoOaistoreCache = mongoOaistoreCache; |
|
| 294 |
} |
|
| 295 |
|
|
| 296 |
public Cache getMongoOaistoreCacheByMdPrefix() {
|
|
| 297 |
return this.mongoOaistoreCacheByMdPrefix; |
|
| 298 |
} |
|
| 299 |
|
|
| 300 |
@Required |
|
| 301 |
public void setMongoOaistoreCacheByMdPrefix(final Cache mongoOaistoreCacheByMdPrefix) {
|
|
| 302 |
this.mongoOaistoreCacheByMdPrefix = mongoOaistoreCacheByMdPrefix; |
|
| 303 |
} |
|
| 304 |
|
|
| 265 | 305 |
} |
Also available in: Unified diff
oai import