Revision 60864
Added by Michele Artini about 3 years ago
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/EnrichDatasetsPlugin.java | ||
|---|---|---|
| 9 | 9 |
import org.apache.commons.lang3.StringUtils; |
| 10 | 10 |
import org.apache.commons.logging.Log; |
| 11 | 11 |
import org.apache.commons.logging.LogFactory; |
| 12 |
import org.springframework.beans.factory.annotation.Autowired; |
|
| 12 | 13 |
import org.springframework.beans.factory.annotation.Value; |
| 13 | 14 |
|
| 14 | 15 |
import com.google.gson.Gson; |
| 15 | 16 |
import com.google.gson.reflect.TypeToken; |
| 17 |
import com.mongodb.MongoClient; |
|
| 18 |
import com.mongodb.client.MongoCollection; |
|
| 16 | 19 |
|
| 17 | 20 |
import eu.dnetlib.data.mdstore.plugins.objects.Dataset; |
| 18 | 21 |
import eu.dnetlib.data.mdstore.plugins.objects.MdRecord; |
| ... | ... | |
| 29 | 32 |
@Value("${plugin.enrich.dataset.dli.url}")
|
| 30 | 33 |
private String baseUrl; |
| 31 | 34 |
|
| 35 |
@Autowired |
|
| 36 |
private MongoClient mongoClient; |
|
| 37 |
|
|
| 32 | 38 |
private Counter enrichDatasetsCounter = new Counter(); |
| 33 | 39 |
|
| 34 | 40 |
@Override |
| ... | ... | |
| 48 | 54 |
} |
| 49 | 55 |
|
| 50 | 56 |
@Override |
| 57 |
protected MongoCollection<org.bson.Document> getCacheCollection() {
|
|
| 58 |
return mongoClient.getDatabase("API_CACHES").getCollection("DATASETS_API_CACHE");
|
|
| 59 |
} |
|
| 60 |
|
|
| 61 |
@Override |
|
| 51 | 62 |
protected boolean updateDocument(final MdRecord doc, final String response) {
|
| 52 | 63 |
|
| 53 | 64 |
doc.getDatasets().clear(); |
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/GenericDoiMdstorePlugin.java | ||
|---|---|---|
| 5 | 5 |
|
| 6 | 6 |
import org.apache.commons.logging.Log; |
| 7 | 7 |
import org.apache.commons.logging.LogFactory; |
| 8 |
import org.bson.Document; |
|
| 8 | 9 |
|
| 10 |
import com.mongodb.BasicDBObject; |
|
| 11 |
import com.mongodb.client.MongoCollection; |
|
| 12 |
|
|
| 9 | 13 |
import eu.dnetlib.data.mdstore.plugins.objects.MdRecord; |
| 10 | 14 |
import eu.dnetlib.data.utils.HttpFetcher; |
| 11 | 15 |
|
| ... | ... | |
| 17 | 21 |
protected final boolean updateRecord(final String recordId, final MdRecord record) {
|
| 18 | 22 |
for (final String doi : record.getDois()) {
|
| 19 | 23 |
log.debug(" Record " + record.getId() + " has doi " + doi);
|
| 20 |
final String response = download(doi);
|
|
| 24 |
final String response = obtainData(doi);
|
|
| 21 | 25 |
if ((response != null) && updateDocument(record, response)) { return true; }
|
| 22 | 26 |
} |
| 23 | 27 |
return false; |
| ... | ... | |
| 27 | 31 |
|
| 28 | 32 |
abstract protected URI prepareURI(String doi) throws URISyntaxException; |
| 29 | 33 |
|
| 34 |
abstract protected MongoCollection<Document> getCacheCollection(); |
|
| 35 |
|
|
| 36 |
private String obtainData(final String doi) {
|
|
| 37 |
final MongoCollection<Document> cacheColl = getCacheCollection(); |
|
| 38 |
|
|
| 39 |
final Document row = cacheColl.find(new BasicDBObject("doi", doi)).first();
|
|
| 40 |
if (row != null) {
|
|
| 41 |
return row.get("data") != null ? row.get("data").toString() : null;
|
|
| 42 |
} else {
|
|
| 43 |
final String res = download(doi); |
|
| 44 |
|
|
| 45 |
final Document doc = new Document(); |
|
| 46 |
doc.append("doi", doi);
|
|
| 47 |
doc.append("data", res);
|
|
| 48 |
cacheColl.insertOne(doc); |
|
| 49 |
|
|
| 50 |
return res; |
|
| 51 |
} |
|
| 52 |
} |
|
| 53 |
|
|
| 30 | 54 |
private String download(final String doi) {
|
| 31 | 55 |
try {
|
| 32 | 56 |
return HttpFetcher.fetch(prepareURI(doi)); |
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/EnrichOpenairePlugin.java | ||
|---|---|---|
| 19 | 19 |
import org.dom4j.Element; |
| 20 | 20 |
import org.dom4j.Node; |
| 21 | 21 |
import org.dom4j.io.SAXReader; |
| 22 |
import org.springframework.beans.factory.annotation.Autowired; |
|
| 22 | 23 |
import org.springframework.beans.factory.annotation.Value; |
| 23 | 24 |
|
| 24 | 25 |
import com.google.common.base.Splitter; |
| 26 |
import com.mongodb.MongoClient; |
|
| 27 |
import com.mongodb.client.MongoCollection; |
|
| 25 | 28 |
|
| 26 | 29 |
import eu.dnetlib.data.mdstore.plugins.objects.MdRecord; |
| 27 | 30 |
import eu.dnetlib.data.mdstore.plugins.objects.MyURL; |
| ... | ... | |
| 37 | 40 |
@Value("${plugin.enrich.openaire.datasources.blacklist}")
|
| 38 | 41 |
private String datasourceBlackList; |
| 39 | 42 |
|
| 43 |
@Autowired |
|
| 44 |
private MongoClient mongoClient; |
|
| 45 |
|
|
| 40 | 46 |
private Map<String, Counter> counters = new HashMap<>(); |
| 41 | 47 |
|
| 42 | 48 |
@Override |
| ... | ... | |
| 45 | 51 |
} |
| 46 | 52 |
|
| 47 | 53 |
@Override |
| 54 |
protected MongoCollection<org.bson.Document> getCacheCollection() {
|
|
| 55 |
return mongoClient.getDatabase("API_CACHES").getCollection("OPENAIRE_API_CACHE");
|
|
| 56 |
} |
|
| 57 |
|
|
| 58 |
@Override |
|
| 48 | 59 |
protected void reconfigure(final Map<String, String> params) {
|
| 49 | 60 |
counters.clear(); |
| 50 | 61 |
counters.put("subjects", new Counter());
|
Also available in: Unified diff
API Cache