Revision 45499
Added by Alessia Bardi over 7 years ago
| modules/dnet-data-provision-services/trunk/src/test/java/eu/dnetlib/oai/mongo/MongoPublisherStoreTest.java | ||
|---|---|---|
| 9 | 9 |
import org.junit.Assert; |
| 10 | 10 |
import org.junit.Test; |
| 11 | 11 |
|
| 12 |
import eu.dnetlib.oai.mongo.MongoPublisherStore; |
|
| 13 |
import eu.dnetlib.oai.mongo.RecordInfoGenerator; |
|
| 14 |
|
|
| 15 | 12 |
/** |
| 16 | 13 |
* @author sandro |
| 17 | 14 |
* |
| ... | ... | |
| 19 | 16 |
public class MongoPublisherStoreTest {
|
| 20 | 17 |
|
| 21 | 18 |
/** |
| 22 |
* Test method for {@link eu.dnetlib.data.oai.store.mongo.MongoPublisherStore#createCompressRecord(java.lang.String)}.
|
|
| 19 |
* Test method for {@link eu.dnetlib.oai.mongo.MongoPublisherStore#createCompressRecord(java.lang.String)}.
|
|
| 23 | 20 |
* |
| 24 | 21 |
* @throws IOException |
| 25 | 22 |
*/ |
| modules/dnet-data-provision-services/trunk/src/main/java/eu/dnetlib/oai/mongo/MongoPublisherStore.java | ||
|---|---|---|
| 13 | 13 |
import java.util.zip.ZipEntry; |
| 14 | 14 |
import java.util.zip.ZipOutputStream; |
| 15 | 15 |
|
| 16 |
import com.google.common.base.Predicate; |
|
| 17 | 16 |
import com.google.common.base.Stopwatch; |
| 18 | 17 |
import com.google.common.collect.Iterables; |
| 19 | 18 |
import com.google.common.collect.Lists; |
| ... | ... | |
| 121 | 120 |
public RecordInfo getRecord(final String recordId) {
|
| 122 | 121 |
final Bson query = Filters.eq(OAIConfigurationReader.ID_FIELD, recordId); |
| 123 | 122 |
final DBObject result = this.collection.find(query).first(); |
| 124 |
log.debug(result); |
|
| 125 | 123 |
return this.recordInfoGenerator.transformDBObject(result, true); |
| 126 | 124 |
} |
| 127 | 125 |
|
| ... | ... | |
| 377 | 375 |
final PublisherRecordParser parser = new PublisherRecordParser(this.mongoFields); |
| 378 | 376 |
log.debug("configured parser for fields: "+this.mongoFields);
|
| 379 | 377 |
final Multimap<String, String> recordProperties = parser.parseRecord(record, source); |
| 380 |
String id = "";
|
|
| 381 |
String oaiID = "";
|
|
| 378 |
String id; |
|
| 379 |
String oaiID; |
|
| 382 | 380 |
if (recordProperties.containsKey(OAIConfigurationReader.ID_FIELD)) {
|
| 383 | 381 |
id = recordProperties.get(OAIConfigurationReader.ID_FIELD).iterator().next(); |
| 384 | 382 |
oaiID = getOAIIdentifier(id); |
| ... | ... | |
| 395 | 393 |
} |
| 396 | 394 |
} else {
|
| 397 | 395 |
log.error("parsed record seems invalid -- no identifier property with name: " + OAIConfigurationReader.ID_FIELD);
|
| 398 |
log.debug("Extracted property map: \n"+recordProperties);
|
|
| 396 |
log.error("Extracted property map: \n"+recordProperties);
|
|
| 399 | 397 |
log.debug("from: \n"+record);
|
| 400 | 398 |
this.discardedCollection |
| 401 | 399 |
.insertOne(new BasicDBObject(OAIConfigurationReader.SET_FIELD, source).append(OAIConfigurationReader.BODY_FIELD, record).append( |
| ... | ... | |
| 415 | 413 |
|
| 416 | 414 |
final Iterable<String> setSpecs = |
| 417 | 415 |
values.stream().map(s -> MongoPublisherStore.this.mongoSetCollection.normalizeSetSpec(s)).collect(Collectors.toList()); |
| 418 |
|
|
| 419 | 416 |
obj.put(key, setSpecs); |
| 420 | 417 |
} else {
|
| 421 | 418 |
// let's check if the key is the name of a repeatable field or not |
| 422 |
final PublisherField keyField = Iterables.find(this.mongoFields, new Predicate<PublisherField>() {
|
|
| 423 |
|
|
| 424 |
@Override |
|
| 425 |
public boolean apply(final PublisherField field) {
|
|
| 426 |
return field.getFieldName().equals(key); |
|
| 427 |
} |
|
| 428 |
}, null); |
|
| 419 |
final PublisherField keyField = Iterables.find(this.mongoFields, field -> field.getFieldName().equals(key), null); |
|
| 429 | 420 |
if (keyField == null) {
|
| 430 | 421 |
log.warn("Expected field to index: " + key + " could not be found, but we keep going...");
|
| 431 | 422 |
} |
| modules/dnet-data-provision-services/trunk/src/main/java/eu/dnetlib/oai/actions/SyncAction.java | ||
|---|---|---|
| 27 | 27 |
|
| 28 | 28 |
@Override |
| 29 | 29 |
public void execute(final BlackboardServerHandler handler, final BlackboardJob job) throws Exception {
|
| 30 |
if(log.isDebugEnabled()) {
|
|
| 31 |
log.debug("Job parameters:");
|
|
| 32 |
job.getParameters().forEach((k, v) -> log.debug("Name : " + k + " Value : " + v));
|
|
| 33 |
} |
|
| 34 |
|
|
| 30 | 35 |
String epr = job.getParameters().get(OAIParameterNames.OAI_SYNC_EPR); |
| 31 | 36 |
String recordSource = job.getParameters().get(OAIParameterNames.OAI_SOURCE); |
| 32 | 37 |
boolean alwaysNewRecord = Boolean.valueOf(job.getParameters().get(OAIParameterNames.OAI_ALWAYS_NEW_RECORD)); |
| modules/dnet-data-provision-services/trunk/src/main/java/eu/dnetlib/oai/sync/OAIStoreSynchronizer.java | ||
|---|---|---|
| 32 | 32 |
final Callable<?> failCallback) {
|
| 33 | 33 |
try {
|
| 34 | 34 |
log.fatal("Synchronizing content for source metadata format " + sourceMetadataFormat);
|
| 35 |
log.fatal("Record source: "+recordSource);
|
|
| 35 | 36 |
MongoPublisherStore store = this.getStore(sourceMetadataFormat, dbName, alwaysNewRecord); |
| 36 | 37 |
int count = store.feed(records, recordSource); |
| 37 | 38 |
|
Also available in: Unified diff
Basically adjusted logs for debugging