Revision 49368
Added by Michele Artini over 6 years ago
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/EnrichDatasetsPlugin.java | ||
|---|---|---|
| 2 | 2 |
|
| 3 | 3 |
import java.net.URI; |
| 4 | 4 |
import java.net.URISyntaxException; |
| 5 |
import java.util.HashMap; |
|
| 6 |
import java.util.List; |
|
| 7 |
import java.util.Map; |
|
| 5 | 8 |
|
| 9 |
import org.apache.commons.lang3.StringUtils; |
|
| 10 |
import org.apache.commons.logging.Log; |
|
| 11 |
import org.apache.commons.logging.LogFactory; |
|
| 6 | 12 |
import org.dom4j.Document; |
| 13 |
import org.dom4j.Element; |
|
| 14 |
import org.dom4j.Namespace; |
|
| 15 |
import org.dom4j.QName; |
|
| 16 |
import org.springframework.beans.factory.annotation.Value; |
|
| 7 | 17 |
|
| 18 |
import com.google.gson.Gson; |
|
| 19 |
import com.google.gson.reflect.TypeToken; |
|
| 20 |
|
|
| 21 |
import eu.dnetlib.data.mdstore.plugins.objects.dli.DliIdentifier; |
|
| 22 |
import eu.dnetlib.data.mdstore.plugins.objects.dli.DliRelation; |
|
| 23 |
|
|
| 8 | 24 |
public class EnrichDatasetsPlugin extends GenericDoiMdstorePlugin {
|
| 9 | 25 |
|
| 26 |
private static final Log log = LogFactory.getLog(EnrichDatasetsPlugin.class); |
|
| 27 |
|
|
| 28 |
@Value("${plugin.enrich.dataset.dli.url}")
|
|
| 29 |
private String baseUrl; |
|
| 30 |
|
|
| 10 | 31 |
@Override |
| 11 | 32 |
protected URI prepareURI(final String doi) throws URISyntaxException {
|
| 12 |
return new URI("https://api-dliservice-prototype-dli.d4science.org/v1/linksFromPid?pid=" + doi);
|
|
| 33 |
return new URI(String.format(baseUrl, doi));
|
|
| 13 | 34 |
} |
| 14 | 35 |
|
| 15 | 36 |
@Override |
| 16 | 37 |
protected boolean updateDocument(final Document doc, final String response) {
|
| 17 | 38 |
|
| 18 |
System.out.println("**********");
|
|
| 19 |
System.out.println(response); |
|
| 20 |
System.out.println("**********");
|
|
| 39 |
final Gson gson = new Gson(); |
|
| 40 |
final List<DliRelation> rels = gson.fromJson(response, new TypeToken<List<DliRelation>>() {}.getType());
|
|
| 21 | 41 |
|
| 22 |
return response.trim().length() > 10;
|
|
| 42 |
final Map<String, String> datasets = new HashMap<>();
|
|
| 23 | 43 |
|
| 44 |
for (final DliRelation rel : rels) {
|
|
| 45 |
final String title = rel.getTarget().getTitle(); |
|
| 46 |
for (final DliIdentifier id : rel.getTarget().getIdentifiers()) {
|
|
| 47 |
if (id.getSchema().equalsIgnoreCase("doi") && StringUtils.isNoneBlank(id.getIdentifier()) && StringUtils.isNotBlank(title)) {
|
|
| 48 |
datasets.put(id.getIdentifier(), title); |
|
| 49 |
} |
|
| 50 |
} |
|
| 51 |
} |
|
| 52 |
|
|
| 53 |
if (datasets.isEmpty()) { return false; }
|
|
| 54 |
|
|
| 55 |
final Element node = (Element) doc.selectSingleNode("//*[local-name() = 'datasets']");
|
|
| 56 |
|
|
| 57 |
datasets.entrySet().forEach(e -> {
|
|
| 58 |
final Element ds = node.addElement(new QName("dataset", new Namespace("isti", "http://www.isti.cnr.it/")));
|
|
| 59 |
ds.addAttribute("doi", e.getKey());
|
|
| 60 |
ds.addAttribute("url", "https://dx.doi.org/" + e.getKey());
|
|
| 61 |
ds.setText(e.getValue()); |
|
| 62 |
log.info("Found dataset: " + e.getKey() + " -> " + e.getValue());
|
|
| 63 |
}); |
|
| 64 |
|
|
| 65 |
return true; |
|
| 66 |
|
|
| 24 | 67 |
} |
| 25 | 68 |
} |
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/EnrichProjectsPlugin.java | ||
|---|---|---|
| 15 | 15 |
import org.dom4j.DocumentException; |
| 16 | 16 |
import org.dom4j.Element; |
| 17 | 17 |
import org.dom4j.io.SAXReader; |
| 18 |
import org.springframework.beans.factory.annotation.Value; |
|
| 18 | 19 |
|
| 19 | 20 |
import com.mongodb.BasicDBObject; |
| 20 | 21 |
import com.mongodb.DBObject; |
| ... | ... | |
| 28 | 29 |
|
| 29 | 30 |
private static final Log log = LogFactory.getLog(EnrichProjectsPlugin.class); |
| 30 | 31 |
|
| 32 |
@Value("${plugin.enrich.projects.openaire.url}")
|
|
| 33 |
private String baseUrl; |
|
| 34 |
|
|
| 31 | 35 |
@Override |
| 32 | 36 |
public void process(final MongoMDStore store, final Map<String, String> params) throws MDStoreServiceException {
|
| 33 | 37 |
log.info("*****************************************");
|
| ... | ... | |
| 118 | 122 |
private Project resolveProject(final String s) {
|
| 119 | 123 |
try {
|
| 120 | 124 |
final String[] arr = s.split("/");
|
| 121 |
final String url = String.format("http://api.openaire.eu/search/projects?funder=%s&fundingStream=%s&grantID=%s", arr[0], arr[1], arr[2]);
|
|
| 125 |
final String url = String.format(baseUrl, arr[0], arr[1], arr[2]);
|
|
| 122 | 126 |
return Project.newInstance(new URI(url)); |
| 123 | 127 |
} catch (final URISyntaxException e) {
|
| 124 | 128 |
log.error("Error resolving project: " + s, e);
|
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/GenericDoiMdstorePlugin.java | ||
|---|---|---|
| 41 | 41 |
try {
|
| 42 | 42 |
sslContextBuilder.loadTrustMaterial(null, (chain, authType) -> true); |
| 43 | 43 |
sslSocketFactory = new SSLConnectionSocketFactory(sslContextBuilder.build()); |
| 44 |
} catch (final NoSuchAlgorithmException e) {
|
|
| 45 |
// TODO Auto-generated catch block |
|
| 46 |
e.printStackTrace(); |
|
| 47 |
} catch (final KeyStoreException e) {
|
|
| 48 |
// TODO Auto-generated catch block |
|
| 49 |
e.printStackTrace(); |
|
| 50 |
} catch (final KeyManagementException e) {
|
|
| 51 |
// TODO Auto-generated catch block |
|
| 52 |
e.printStackTrace(); |
|
| 44 |
} catch (final NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) {
|
|
| 45 |
log.error(e);; |
|
| 53 | 46 |
} |
| 54 | 47 |
} |
| 55 | 48 |
|
| ... | ... | |
| 78 | 71 |
for (final Object o : doc.selectNodes("//*[local-name()='alternateIdentifier' and @alternateIdentifierType='doi']")) {
|
| 79 | 72 |
final String doi = ((Node) o).getText().trim(); |
| 80 | 73 |
|
| 81 |
log.info(" Record " + recordId + " has doi " + doi);
|
|
| 74 |
log.debug(" Record " + recordId + " has doi " + doi);
|
|
| 82 | 75 |
final String response = download(doi); |
| 83 | 76 |
if ((response != null) && updateDocument(doc, response)) {
|
| 84 | 77 |
collPubs.updateOne(new BasicDBObject("id", recordId), new BasicDBObject("$set", new BasicDBObject("body", doc.asXML())));
|
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/EnrichLabsPlugin.java | ||
|---|---|---|
| 20 | 20 |
import org.dom4j.Element; |
| 21 | 21 |
import org.dom4j.Node; |
| 22 | 22 |
import org.dom4j.io.SAXReader; |
| 23 |
import org.springframework.beans.factory.annotation.Value; |
|
| 23 | 24 |
|
| 24 | 25 |
import com.google.common.base.Splitter; |
| 25 | 26 |
import com.google.common.collect.Lists; |
| ... | ... | |
| 40 | 41 |
|
| 41 | 42 |
private static final Log log = LogFactory.getLog(EnrichLabsPlugin.class); |
| 42 | 43 |
|
| 44 |
@Value("${plugin.enrich.labs.pimpa.url}")
|
|
| 45 |
private String serviceUrl; |
|
| 46 |
|
|
| 43 | 47 |
@SuppressWarnings("unchecked")
|
| 44 | 48 |
@Override |
| 45 | 49 |
public void process(final MongoMDStore store, final Map<String, String> params) throws MDStoreServiceException {
|
| ... | ... | |
| 210 | 214 |
private DataRange findDataRange(final int from, final int to) {
|
| 211 | 215 |
final JaxWsProxyFactoryBean factory = new JaxWsProxyFactoryBean(); |
| 212 | 216 |
factory.setServiceClass(PimpaService.class); |
| 213 |
factory.setAddress("http://pimpa.isti.cnr.it/PERSONALE/web-services/iop/iop.webservice.php");
|
|
| 217 |
factory.setAddress(serviceUrl);
|
|
| 214 | 218 |
final PimpaService pimpaService = (PimpaService) factory.create(); |
| 215 | 219 |
return pimpaService.getDataRange(from, to); |
| 216 | 220 |
} |
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/objects/dli/DliObjectType.java | ||
|---|---|---|
| 1 |
|
|
| 2 |
package eu.dnetlib.data.mdstore.plugins.objects.dli; |
|
| 3 |
|
|
| 4 |
public class DliObjectType {
|
|
| 5 |
|
|
| 6 |
private String subType; |
|
| 7 |
private String type; |
|
| 8 |
|
|
| 9 |
public String getSubType() {
|
|
| 10 |
return subType; |
|
| 11 |
} |
|
| 12 |
|
|
| 13 |
public void setSubType(final String subType) {
|
|
| 14 |
this.subType = subType; |
|
| 15 |
} |
|
| 16 |
|
|
| 17 |
public String getType() {
|
|
| 18 |
return type; |
|
| 19 |
} |
|
| 20 |
|
|
| 21 |
public void setType(final String type) {
|
|
| 22 |
this.type = type; |
|
| 23 |
} |
|
| 24 |
|
|
| 25 |
} |
|
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/objects/dli/DliProvider.java | ||
|---|---|---|
| 1 |
|
|
| 2 |
package eu.dnetlib.data.mdstore.plugins.objects.dli; |
|
| 3 |
|
|
| 4 |
import java.util.List; |
|
| 5 |
|
|
| 6 |
public class DliProvider {
|
|
| 7 |
|
|
| 8 |
private List<DliIdentifier> identifiers = null; |
|
| 9 |
private String name; |
|
| 10 |
|
|
| 11 |
public List<DliIdentifier> getIdentifiers() {
|
|
| 12 |
return identifiers; |
|
| 13 |
} |
|
| 14 |
|
|
| 15 |
public void setIdentifiers(final List<DliIdentifier> identifiers) {
|
|
| 16 |
this.identifiers = identifiers; |
|
| 17 |
} |
|
| 18 |
|
|
| 19 |
public String getName() {
|
|
| 20 |
return name; |
|
| 21 |
} |
|
| 22 |
|
|
| 23 |
public void setName(final String name) {
|
|
| 24 |
this.name = name; |
|
| 25 |
} |
|
| 26 |
|
|
| 27 |
} |
|
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/objects/dli/DliEntity.java | ||
|---|---|---|
| 1 |
|
|
| 2 |
package eu.dnetlib.data.mdstore.plugins.objects.dli; |
|
| 3 |
|
|
| 4 |
import java.util.List; |
|
| 5 |
|
|
| 6 |
public class DliEntity {
|
|
| 7 |
|
|
| 8 |
private List<DliIdentifier> identifiers = null; |
|
| 9 |
private List<DliProvider> objectProvider = null; |
|
| 10 |
private DliObjectType objectType; |
|
| 11 |
private List<DliPublisher> publisher = null; |
|
| 12 |
private String title; |
|
| 13 |
|
|
| 14 |
public List<DliIdentifier> getIdentifiers() {
|
|
| 15 |
return identifiers; |
|
| 16 |
} |
|
| 17 |
|
|
| 18 |
public void setIdentifiers(final List<DliIdentifier> identifiers) {
|
|
| 19 |
this.identifiers = identifiers; |
|
| 20 |
} |
|
| 21 |
|
|
| 22 |
public List<DliProvider> getObjectProvider() {
|
|
| 23 |
return objectProvider; |
|
| 24 |
} |
|
| 25 |
|
|
| 26 |
public void setObjectProvider(final List<DliProvider> objectProvider) {
|
|
| 27 |
this.objectProvider = objectProvider; |
|
| 28 |
} |
|
| 29 |
|
|
| 30 |
public DliObjectType getObjectType() {
|
|
| 31 |
return objectType; |
|
| 32 |
} |
|
| 33 |
|
|
| 34 |
public void setObjectType(final DliObjectType objectType) {
|
|
| 35 |
this.objectType = objectType; |
|
| 36 |
} |
|
| 37 |
|
|
| 38 |
public List<DliPublisher> getPublisher() {
|
|
| 39 |
return publisher; |
|
| 40 |
} |
|
| 41 |
|
|
| 42 |
public void setPublisher(final List<DliPublisher> publisher) {
|
|
| 43 |
this.publisher = publisher; |
|
| 44 |
} |
|
| 45 |
|
|
| 46 |
public String getTitle() {
|
|
| 47 |
return title; |
|
| 48 |
} |
|
| 49 |
|
|
| 50 |
public void setTitle(final String title) {
|
|
| 51 |
this.title = title; |
|
| 52 |
} |
|
| 53 |
|
|
| 54 |
} |
|
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/objects/dli/DliPublisher.java | ||
|---|---|---|
| 1 |
|
|
| 2 |
package eu.dnetlib.data.mdstore.plugins.objects.dli; |
|
| 3 |
|
|
| 4 |
public class DliPublisher {
|
|
| 5 |
|
|
| 6 |
private String name; |
|
| 7 |
|
|
| 8 |
public String getName() {
|
|
| 9 |
return name; |
|
| 10 |
} |
|
| 11 |
|
|
| 12 |
public void setName(final String name) {
|
|
| 13 |
this.name = name; |
|
| 14 |
} |
|
| 15 |
|
|
| 16 |
} |
|
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/objects/dli/DliRelation.java | ||
|---|---|---|
| 1 |
|
|
| 2 |
package eu.dnetlib.data.mdstore.plugins.objects.dli; |
|
| 3 |
|
|
| 4 |
import java.util.List; |
|
| 5 |
|
|
| 6 |
public class DliRelation {
|
|
| 7 |
|
|
| 8 |
private List<DliProvider> linkProvider = null; |
|
| 9 |
private DliRelationship relationship; |
|
| 10 |
private DliEntity source; |
|
| 11 |
private DliEntity target; |
|
| 12 |
|
|
| 13 |
public List<DliProvider> getLinkProvider() {
|
|
| 14 |
return linkProvider; |
|
| 15 |
} |
|
| 16 |
|
|
| 17 |
public void setLinkProvider(final List<DliProvider> linkProvider) {
|
|
| 18 |
this.linkProvider = linkProvider; |
|
| 19 |
} |
|
| 20 |
|
|
| 21 |
public DliRelationship getRelationship() {
|
|
| 22 |
return relationship; |
|
| 23 |
} |
|
| 24 |
|
|
| 25 |
public void setRelationship(final DliRelationship relationship) {
|
|
| 26 |
this.relationship = relationship; |
|
| 27 |
} |
|
| 28 |
|
|
| 29 |
public DliEntity getSource() {
|
|
| 30 |
return source; |
|
| 31 |
} |
|
| 32 |
|
|
| 33 |
public void setSource(final DliEntity source) {
|
|
| 34 |
this.source = source; |
|
| 35 |
} |
|
| 36 |
|
|
| 37 |
public DliEntity getTarget() {
|
|
| 38 |
return target; |
|
| 39 |
} |
|
| 40 |
|
|
| 41 |
public void setTarget(final DliEntity target) {
|
|
| 42 |
this.target = target; |
|
| 43 |
} |
|
| 44 |
|
|
| 45 |
} |
|
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/objects/dli/DliRelationship.java | ||
|---|---|---|
| 1 |
|
|
| 2 |
package eu.dnetlib.data.mdstore.plugins.objects.dli; |
|
| 3 |
|
|
| 4 |
public class DliRelationship {
|
|
| 5 |
|
|
| 6 |
private String inverseRelationship; |
|
| 7 |
private String name; |
|
| 8 |
private String schema; |
|
| 9 |
|
|
| 10 |
public String getInverseRelationship() {
|
|
| 11 |
return inverseRelationship; |
|
| 12 |
} |
|
| 13 |
|
|
| 14 |
public void setInverseRelationship(final String inverseRelationship) {
|
|
| 15 |
this.inverseRelationship = inverseRelationship; |
|
| 16 |
} |
|
| 17 |
|
|
| 18 |
public String getName() {
|
|
| 19 |
return name; |
|
| 20 |
} |
|
| 21 |
|
|
| 22 |
public void setName(final String name) {
|
|
| 23 |
this.name = name; |
|
| 24 |
} |
|
| 25 |
|
|
| 26 |
public String getSchema() {
|
|
| 27 |
return schema; |
|
| 28 |
} |
|
| 29 |
|
|
| 30 |
public void setSchema(final String schema) {
|
|
| 31 |
this.schema = schema; |
|
| 32 |
} |
|
| 33 |
|
|
| 34 |
} |
|
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/objects/dli/DliIdentifier.java | ||
|---|---|---|
| 1 |
|
|
| 2 |
package eu.dnetlib.data.mdstore.plugins.objects.dli; |
|
| 3 |
|
|
| 4 |
public class DliIdentifier {
|
|
| 5 |
|
|
| 6 |
private String identifier; |
|
| 7 |
private String schema; |
|
| 8 |
|
|
| 9 |
public String getIdentifier() {
|
|
| 10 |
return identifier; |
|
| 11 |
} |
|
| 12 |
|
|
| 13 |
public void setIdentifier(final String identifier) {
|
|
| 14 |
this.identifier = identifier; |
|
| 15 |
} |
|
| 16 |
|
|
| 17 |
public String getSchema() {
|
|
| 18 |
return schema; |
|
| 19 |
} |
|
| 20 |
|
|
| 21 |
public void setSchema(final String schema) {
|
|
| 22 |
this.schema = schema; |
|
| 23 |
} |
|
| 24 |
|
|
| 25 |
} |
|
| modules/dnet-isti/trunk/src/main/java/eu/dnetlib/data/mdstore/plugins/EnrichOpenairePlugin.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.data.mdstore.plugins; |
| 2 | 2 |
|
| 3 |
import java.io.StringReader; |
|
| 3 | 4 |
import java.net.URI; |
| 4 | 5 |
import java.net.URISyntaxException; |
| 6 |
import java.util.List; |
|
| 5 | 7 |
|
| 8 |
import org.apache.commons.logging.Log; |
|
| 9 |
import org.apache.commons.logging.LogFactory; |
|
| 6 | 10 |
import org.dom4j.Document; |
| 11 |
import org.dom4j.DocumentException; |
|
| 12 |
import org.dom4j.Node; |
|
| 13 |
import org.dom4j.io.SAXReader; |
|
| 14 |
import org.springframework.beans.factory.annotation.Value; |
|
| 7 | 15 |
|
| 8 | 16 |
public class EnrichOpenairePlugin extends GenericDoiMdstorePlugin {
|
| 9 | 17 |
|
| 18 |
private static final Log log = LogFactory.getLog(EnrichOpenairePlugin.class); |
|
| 19 |
|
|
| 20 |
@Value("${plugin.enrich.publications.openaire.url}")
|
|
| 21 |
private String baseUrl; |
|
| 22 |
|
|
| 10 | 23 |
@Override |
| 11 | 24 |
protected URI prepareURI(final String doi) throws URISyntaxException {
|
| 12 |
return new URI("http://api.openaire.eu/search/publications?doi=" + doi);
|
|
| 25 |
return new URI(String.format(baseUrl, doi));
|
|
| 13 | 26 |
} |
| 14 | 27 |
|
| 15 | 28 |
@Override |
| 16 | 29 |
protected boolean updateDocument(final Document doc, final String response) {
|
| 17 | 30 |
|
| 18 |
System.out.println("**********");
|
|
| 19 |
System.out.println(response); |
|
| 20 |
System.out.println("**********");
|
|
| 31 |
try {
|
|
| 32 |
final Document docRes = (new SAXReader()).read(new StringReader(response)); |
|
| 21 | 33 |
|
| 22 |
return true;
|
|
| 34 |
final List<?> results = docRes.selectNodes("/response/results/result");
|
|
| 23 | 35 |
|
| 36 |
if (results.size() == 1) {
|
|
| 37 |
final Node n = (Node) results.get(0); |
|
| 38 |
System.out.println("------------");
|
|
| 39 |
System.out.println("TITLE 1: " + doc.valueOf("//*[local-name() = 'title']"));
|
|
| 40 |
System.out.println("TITLE 2: " + docRes.valueOf("//*[local-name() = 'title' and @classid='main title']"));
|
|
| 41 |
|
|
| 42 |
for (final Object oid : n.selectNodes(".//originalId")) {
|
|
| 43 |
System.out.println(" - " + ((Node) oid).getText());
|
|
| 44 |
} |
|
| 45 |
return true; |
|
| 46 |
} else if (results.size() == 1) {
|
|
| 47 |
log.warn("Too many responses");
|
|
| 48 |
} |
|
| 49 |
|
|
| 50 |
} catch (final DocumentException e) {
|
|
| 51 |
log.warn("Invalid response", e);
|
|
| 52 |
} |
|
| 53 |
|
|
| 54 |
return false; |
|
| 55 |
|
|
| 24 | 56 |
} |
| 25 | 57 |
} |
| modules/dnet-isti/trunk/src/main/resources/eu/dnetlib/isti/applicationContext-isti.properties | ||
|---|---|---|
| 1 |
plugin.enrich.dataset.dli.url = https://api-dliservice-prototype-dli.d4science.org/v1/linksFromPid?pid=%s |
|
| 2 |
plugin.enrich.labs.pimpa.url = http://pimpa.isti.cnr.it/PERSONALE/web-services/iop/iop.webservice.php |
|
| 3 |
plugin.enrich.publications.openaire.url = http://api.openaire.eu/search/publications?doi=%s |
|
| 4 |
plugin.enrich.projects.openaire.url = http://api.openaire.eu/search/projects?funder=%s&fundingStream=%s&grantID=%s |
|
| modules/dnet-isti/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/TransformationRuleDSResources/people2dataciteTransform.xml | ||
|---|---|---|
| 152 | 152 |
</xsl:for-each> |
| 153 | 153 |
</isti:projects> |
| 154 | 154 |
|
| 155 |
<isti:datasets /> |
|
| 156 |
|
|
| 155 | 157 |
<isti:persons> |
| 156 | 158 |
<xsl:for-each select="//dc:relation[starts-with(., 'info:cnr-pdr/author')]"> |
| 157 | 159 |
<isti:person> |
Also available in: Unified diff
datasets