Revision 42184
Added by Michele Artini about 8 years ago
| OAISetHelper.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.oai.conf; |
| 2 | 2 |
|
| 3 | 3 |
import java.util.List; |
| 4 |
|
|
| 4 | 5 |
import javax.annotation.Resource; |
| 5 | 6 |
|
| 6 |
import eu.dnetlib.data.information.oai.publisher.OaiPublisherRuntimeException; |
|
| 7 |
import eu.dnetlib.data.information.oai.publisher.info.MDFInfo; |
|
| 8 |
import eu.dnetlib.data.information.oai.publisher.info.SetInfo; |
|
| 9 |
import eu.dnetlib.data.oai.store.mongo.MongoPublisherStore; |
|
| 10 |
import eu.dnetlib.data.oai.store.mongo.MongoPublisherStoreDAO; |
|
| 11 |
import eu.dnetlib.data.oai.store.sets.MongoSetCollection; |
|
| 12 |
import org.apache.commons.lang.StringUtils; |
|
| 7 |
import org.apache.commons.lang3.StringUtils; |
|
| 13 | 8 |
import org.apache.commons.logging.Log; |
| 14 | 9 |
import org.apache.commons.logging.LogFactory; |
| 15 | 10 |
|
| 11 |
import eu.dnetlib.oai.info.SetInfo; |
|
| 12 |
import eu.dnetlib.oai.mongo.MongoPublisherStore; |
|
| 13 |
import eu.dnetlib.oai.mongo.MongoPublisherStoreDAO; |
|
| 14 |
import eu.dnetlib.oai.sets.MongoSetCollection; |
|
| 15 |
import eu.dnetlib.rmi.provision.MDFInfo; |
|
| 16 |
import eu.dnetlib.rmi.provision.OaiPublisherRuntimeException; |
|
| 17 |
|
|
| 16 | 18 |
public class OAISetHelper {
|
| 17 | 19 |
|
| 18 | 20 |
private static final Log log = LogFactory.getLog(OAISetHelper.class); // NOPMD by marko on 11/24/08 5:02 PM |
| ... | ... | |
| 31 | 33 |
private void loadConfigurationSets(final String dbName) {
|
| 32 | 34 |
log.debug("*****Dropping and re-creating configuration sets******");
|
| 33 | 35 |
this.mongoSetCollection.dropConfigurationSets(dbName); |
| 34 |
List<SetInfo> oaiConfigSets = configurationReader.getSets();
|
|
| 35 |
for (SetInfo setInfo : oaiConfigSets) {
|
|
| 36 |
final List<SetInfo> oaiConfigSets = this.configurationReader.getSets();
|
|
| 37 |
for (final SetInfo setInfo : oaiConfigSets) {
|
|
| 36 | 38 |
this.mongoSetCollection.upsertSet(setInfo, true, dbName); |
| 37 | 39 |
} |
| 38 | 40 |
log.info("Configuration Sets updated succesfully on db: " + dbName);
|
| ... | ... | |
| 44 | 46 |
public void updateAllCounts(final MDFInfo mdFormat, final String dbName) {
|
| 45 | 47 |
this.mongoSetCollection.ensureIndexes(dbName); |
| 46 | 48 |
|
| 47 |
MongoPublisherStore store = getStore(mdFormat, dbName); |
|
| 48 |
if (store == null) throw new OaiPublisherRuntimeException("Can't count elements for not yet created store (" + mdFormat + ") on db " + dbName);
|
|
| 49 |
final MongoPublisherStore store = getStore(mdFormat, dbName);
|
|
| 50 |
if (store == null) { throw new OaiPublisherRuntimeException("Can't count elements for not yet created store (" + mdFormat + ") on db " + dbName); }
|
|
| 49 | 51 |
|
| 50 | 52 |
updateTotalCount(store, mdFormat, dbName); |
| 51 | 53 |
|
| ... | ... | |
| 57 | 59 |
public void updateConfiguredSetsCount(final MDFInfo mdFormat, final String dbName) {
|
| 58 | 60 |
this.mongoSetCollection.ensureIndexes(dbName); |
| 59 | 61 |
|
| 60 |
MongoPublisherStore store = getStore(mdFormat, dbName); |
|
| 61 |
if (store == null) throw new OaiPublisherRuntimeException("Can't count elements for not yet created store (" + mdFormat + ") on db " + dbName);
|
|
| 62 |
final MongoPublisherStore store = getStore(mdFormat, dbName);
|
|
| 63 |
if (store == null) { throw new OaiPublisherRuntimeException("Can't count elements for not yet created store (" + mdFormat + ") on db " + dbName); }
|
|
| 62 | 64 |
|
| 63 | 65 |
updateConfiguredSetsCount(store, mdFormat, dbName); |
| 64 | 66 |
} |
| 65 | 67 |
|
| 66 | 68 |
protected void updateConfiguredSetsCount(final MongoPublisherStore store, final MDFInfo mdFormat, final String dbName) {
|
| 67 |
List<SetInfo> sets = mongoSetCollection.getConfiguredSets(dbName);
|
|
| 69 |
final List<SetInfo> sets = this.mongoSetCollection.getConfiguredSets(dbName);
|
|
| 68 | 70 |
this.updateCountForSets(store, sets, mdFormat, dbName); |
| 69 | 71 |
} |
| 70 | 72 |
|
| 71 | 73 |
protected void updateProvenanceSetsCount(final MongoPublisherStore store, final MDFInfo mdFormat, final String dbName) {
|
| 72 | 74 |
// now we need to get all distinct set names in the store: |
| 73 |
List<String> distinctSetSpecs = store.getDistinctSetNamesFromRecords(); |
|
| 75 |
final List<String> distinctSetSpecs = store.getDistinctSetNamesFromRecords();
|
|
| 74 | 76 |
store.upsertSets(distinctSetSpecs); |
| 75 |
List<SetInfo> sets = mongoSetCollection.getSetsFromData(dbName);
|
|
| 77 |
final List<SetInfo> sets = this.mongoSetCollection.getSetsFromData(dbName);
|
|
| 76 | 78 |
this.updateCountForSets(store, sets, mdFormat, dbName); |
| 77 | 79 |
} |
| 78 | 80 |
|
| 79 | 81 |
protected void updateTotalCount(final MongoPublisherStore store, final MDFInfo mdFormat, final String dbName) {
|
| 80 |
String baseQuery = mdFormat.getBaseQuery(); |
|
| 81 |
int total = store.count(baseQuery); |
|
| 82 |
mongoSetCollection.updateCounts("ALL", mdFormat.getPrefix(), total, dbName);
|
|
| 82 |
final String baseQuery = mdFormat.getBaseQuery();
|
|
| 83 |
final int total = store.count(baseQuery);
|
|
| 84 |
this.mongoSetCollection.updateCounts("ALL", mdFormat.getPrefix(), total, dbName);
|
|
| 83 | 85 |
log.info("Got total for " + mdFormat.getPrefix() + " with query: " + baseQuery + " on db " + dbName);
|
| 84 | 86 |
} |
| 85 | 87 |
|
| 86 | 88 |
protected void updateCountForSets(final MongoPublisherStore store, final List<SetInfo> oaiSets, final MDFInfo mdFormat, final String dbName) {
|
| 87 | 89 |
|
| 88 |
String baseQuery = mdFormat.getBaseQuery(); |
|
| 89 |
boolean hasBaseQuery = !StringUtils.isBlank(baseQuery); |
|
| 90 |
for (SetInfo setInfo : oaiSets) {
|
|
| 90 |
final String baseQuery = mdFormat.getBaseQuery();
|
|
| 91 |
final boolean hasBaseQuery = !StringUtils.isBlank(baseQuery);
|
|
| 92 |
for (final SetInfo setInfo : oaiSets) {
|
|
| 91 | 93 |
String setQuery = "(" + setInfo.getQuery() + ")";
|
| 92 | 94 |
if (hasBaseQuery) {
|
| 93 | 95 |
setQuery += " AND (" + baseQuery + ")";
|
| 94 | 96 |
} |
| 95 | 97 |
log.info("Counting total for " + mdFormat.getPrefix() + " set " + setInfo + " with query: " + setQuery + " on db " + dbName);
|
| 96 |
int setCount = store.count(setQuery); |
|
| 97 |
mongoSetCollection.updateCounts(setInfo.getSetSpec(), mdFormat.getPrefix(), setCount, dbName); |
|
| 98 |
final int setCount = store.count(setQuery);
|
|
| 99 |
this.mongoSetCollection.updateCounts(setInfo.getSetSpec(), mdFormat.getPrefix(), setCount, dbName);
|
|
| 98 | 100 |
} |
| 99 | 101 |
} |
| 100 | 102 |
|
| 101 | 103 |
private MongoPublisherStore getStore(final MDFInfo mdFormat, final String dbName) {
|
| 102 |
String format = mdFormat.getSourceFormatName(); |
|
| 103 |
String layout = mdFormat.getSourceFormatLayout(); |
|
| 104 |
String interpretation = mdFormat.getSourceFormatInterpretation(); |
|
| 105 |
String sourceKey = format + "-" + layout + "-" + interpretation; |
|
| 106 |
MongoPublisherStore store = this.mongoPublisherStoreDAO.getStore(format, interpretation, layout, dbName); |
|
| 104 |
final String format = mdFormat.getSourceFormatName();
|
|
| 105 |
final String layout = mdFormat.getSourceFormatLayout();
|
|
| 106 |
final String interpretation = mdFormat.getSourceFormatInterpretation();
|
|
| 107 |
final String sourceKey = format + "-" + layout + "-" + interpretation;
|
|
| 108 |
final MongoPublisherStore store = this.mongoPublisherStoreDAO.getStore(format, interpretation, layout, dbName);
|
|
| 107 | 109 |
log.info("Got OAI store " + sourceKey + " via metadata prefix " + mdFormat.getPrefix() + " on db " + dbName);
|
| 108 | 110 |
|
| 109 | 111 |
return store; |
| 110 | 112 |
} |
| 111 | 113 |
|
| 112 | 114 |
public OAIConfigurationReader getConfigurationReader() {
|
| 113 |
return configurationReader; |
|
| 115 |
return this.configurationReader;
|
|
| 114 | 116 |
} |
| 115 | 117 |
|
| 116 | 118 |
public void setConfigurationReader(final OAIConfigurationReader configurationReader) {
|
| ... | ... | |
| 118 | 120 |
} |
| 119 | 121 |
|
| 120 | 122 |
public MongoPublisherStoreDAO getMongoPublisherStoreDAO() {
|
| 121 |
return mongoPublisherStoreDAO; |
|
| 123 |
return this.mongoPublisherStoreDAO;
|
|
| 122 | 124 |
} |
| 123 | 125 |
|
| 124 | 126 |
public void setMongoPublisherStoreDAO(final MongoPublisherStoreDAO mongoPublisherStoreDAO) {
|
| ... | ... | |
| 126 | 128 |
} |
| 127 | 129 |
|
| 128 | 130 |
public MongoSetCollection getMongoSetCollection() {
|
| 129 |
return mongoSetCollection; |
|
| 131 |
return this.mongoSetCollection;
|
|
| 130 | 132 |
} |
| 131 | 133 |
|
| 132 | 134 |
public void setMongoSetCollection(final MongoSetCollection mongoSetCollection) {
|
Also available in: Unified diff
oai import