Revision 54287
Added by Sandro La Bruzzo over 5 years ago
modules/dnet-dli/trunk/src/test/java/eu/dnetlib/resolver/DLIResolverTest.java | ||
---|---|---|
79 | 79 |
|
80 | 80 |
|
81 | 81 |
List<String> dois = Arrays.asList( |
82 |
"10.4225/andstest", |
|
83 |
"10.4230/dagrep.1.2.1", |
|
84 |
"10.4230/dagrep.1.2.30", |
|
85 |
"10.1594/wdcc/clm_c20_3_d3", |
|
86 |
"10.4230/dagrep.1.2.67", |
|
87 |
"10.4230/dagrep.1.2.16", |
|
88 |
"10.4230/dagrep.1.2.i", |
|
89 |
"10.4230/dagrep.1.3.1", |
|
90 |
"10.4230/lipics.iclp.2011.162", |
|
91 |
"10.4230/lipics.iclp.2011.187", |
|
92 |
"10.4230/lipics.iclp.2011.1", |
|
93 |
"10.4230/lipics.iclp.2011.198", |
|
94 |
"10.4230/lipics.iclp.2011.209", |
|
95 |
"10.4230/lipics.iclp.2011.i", |
|
96 |
"10.4230/lipics.iclp.2011.220", |
|
97 |
"10.4230/lipics.iclp.2011.128", |
|
98 |
"10.4230/lipics.iclp.2011.11", |
|
99 |
"10.4230/lipics.iclp.2011.231", |
|
100 |
"10.4230/dagrep.1.2.47", |
|
101 |
"10.4230/lipics.iclp.2011.117"); |
|
82 |
"10.1594/pangaea.230638"); |
|
102 | 83 |
|
103 | 84 |
DataciteOfflineResolver resolver = new DataciteOfflineResolver(); |
104 | 85 |
resolver.setMongoClient(client); |
modules/dnet-dli/trunk/src/test/java/eu/dnetlib/dli/collector/plugin/CrossRefIteratorTest.java | ||
---|---|---|
1 |
package eu.dnetlib.dli.collector.plugin; |
|
2 |
|
|
3 |
import org.antlr.stringtemplate.StringTemplate; |
|
4 |
import org.apache.commons.io.IOUtils; |
|
5 |
import org.junit.Test; |
|
6 |
|
|
7 |
import java.io.IOException; |
|
8 |
import java.io.InputStream; |
|
9 |
|
|
10 |
public class CrossRefIteratorTest { |
|
11 |
|
|
12 |
|
|
13 |
@Test |
|
14 |
public void CrossRefIteratorTest() throws IOException { |
|
15 |
final InputStream resourceAsStream = getClass().getResourceAsStream("/eu/dnetlib/dli/templates/Scholixv1.st"); |
|
16 |
|
|
17 |
|
|
18 |
final String template = IOUtils.toString(resourceAsStream); |
|
19 |
final StringTemplate st = new StringTemplate(template); |
|
20 |
|
|
21 |
CrossRefIterator ci = new CrossRefIterator(st); |
|
22 |
int i =0; |
|
23 |
while (ci.hasNext()){ |
|
24 |
ci.next(); |
|
25 |
|
|
26 |
i++; |
|
27 |
if (i % 100 ==0 ) { |
|
28 |
System.out.println("Read "+i); |
|
29 |
} |
|
30 |
} |
|
31 |
} |
|
32 |
} |
modules/dnet-dli/trunk/src/test/resources/eu/dnetlib/dli/templates/Scholixv1.st | ||
---|---|---|
1 |
<link xmlns="http://www.scholix.org"> |
|
2 |
$if(object.date)$<publicationDate>$object.date$</publicationDate>$endif$ |
|
3 |
<linkProvider> |
|
4 |
$object.datasourceProvenance:{ |
|
5 |
<name>$it.datasource$</name> |
|
6 |
<identifiers> |
|
7 |
<identifier>$it.datasourceId$</identifier> |
|
8 |
</identifiers> |
|
9 |
</linkProvider> |
|
10 |
}$ |
|
11 |
$object.relations:{ |
|
12 |
<relashionship> |
|
13 |
<name>$it.relationSemantics$</name> |
|
14 |
<schema>datacite</schema> |
|
15 |
$if(it.inverseRelation)$ |
|
16 |
<inverseRelashionship>$it.inverseRelation$</inverseRelashionship> |
|
17 |
$endif$ |
|
18 |
</relashionship> |
|
19 |
}$ |
|
20 |
<source> |
|
21 |
<identifier> |
|
22 |
<identifier>$object.pid$</identifier> |
|
23 |
<schema>$object.type$</schema> |
|
24 |
</identifier> |
|
25 |
<objectType> |
|
26 |
<type>$object.type$</type> |
|
27 |
</objectType> |
|
28 |
</source> |
|
29 |
<target> |
|
30 |
$object.relations:{ |
|
31 |
<identifier> |
|
32 |
<identifier>$it.targetPID.id$</identifier> |
|
33 |
<schema>$it.targetPID.type$</schema> |
|
34 |
</identifier> |
|
35 |
<objectType> |
|
36 |
<type>$it.targetType$</type> |
|
37 |
</objectType> |
|
38 |
}$ |
|
39 |
</target> |
|
40 |
</link> |
modules/dnet-dli/trunk/src/main/java/eu/dnetlib/resolver/mdstore/plugin/DLIRecordResolver.java | ||
---|---|---|
55 | 55 |
|
56 | 56 |
boolean shouldUpdate = false; |
57 | 57 |
if (inputObject.getCompletionStatus() == null || !inputObject.getCompletionStatus().equals(CompletionStatus.complete.toString())) { |
58 |
shouldUpdate = shouldUpdate || tryToResolveRecord(inputObject);
|
|
58 |
shouldUpdate = tryToResolveRecord(inputObject); |
|
59 | 59 |
} |
60 | 60 |
return resolveRelations(inputRecord, inputObject, shouldUpdate); |
61 | 61 |
|
modules/dnet-dli/trunk/src/main/java/eu/dnetlib/dli/resolver/DataciteOfflineResolver.java | ||
---|---|---|
7 | 7 |
import com.mongodb.client.FindIterable; |
8 | 8 |
import com.mongodb.client.MongoCollection; |
9 | 9 |
import com.mongodb.client.model.Filters; |
10 |
import eu.dnetlib.dli.resolver.model.CompletionStatus; |
|
10 | 11 |
import eu.dnetlib.dli.resolver.model.DLIObjectProvenance; |
11 | 12 |
import eu.dnetlib.dli.resolver.model.DLIResolvedObject; |
12 | 13 |
import eu.dnetlib.enabling.tools.DnetStreamSupport; |
... | ... | |
16 | 17 |
import org.apache.commons.logging.Log; |
17 | 18 |
import org.apache.commons.logging.LogFactory; |
18 | 19 |
import org.bson.Document; |
20 |
import org.springframework.beans.factory.annotation.Required; |
|
19 | 21 |
import org.springframework.beans.factory.annotation.Value; |
20 | 22 |
|
21 | 23 |
|
22 | 24 |
import java.util.List; |
25 |
import java.util.Objects; |
|
23 | 26 |
import java.util.stream.Collectors; |
24 | 27 |
|
25 | 28 |
|
... | ... | |
55 | 58 |
return parseResponse(result); |
56 | 59 |
} |
57 | 60 |
} catch (Throwable e) { |
58 |
log.error("Error on getting item with Pid: " + pid); |
|
61 |
log.error("Error on getting item with Pid: " + pid, e);
|
|
59 | 62 |
} |
60 | 63 |
return null; |
61 | 64 |
} |
... | ... | |
78 | 81 |
dli.setSubjects( |
79 | 82 |
DnetStreamSupport.generateStreamFromIterator(root.getAsJsonArray("subjects").iterator()) |
80 | 83 |
.map(JsonElement::getAsJsonObject) |
81 |
.map(it -> new SubjectType(it.get("scheme").getAsString(), it.get("term").getAsString())) |
|
84 |
.map(this::createSubjectType) |
|
85 |
.filter(Objects::nonNull) |
|
82 | 86 |
.collect(Collectors.toList()) |
83 | 87 |
); |
84 | 88 |
} |
85 |
dli.setDatasourceProvenance( |
|
86 |
DnetStreamSupport.generateStreamFromIterator(root.getAsJsonArray("datasourceProvenance").iterator()) |
|
87 |
.map(JsonElement::getAsJsonObject) |
|
88 |
.map(it -> { |
|
89 |
final DLIObjectProvenance provenance =new DLIObjectProvenance(); |
|
90 |
provenance.setDatasourceId(it.get("datasourceId").getAsString()); |
|
91 |
provenance.setCompletionStatus(it.get("completionStatus").getAsString()); |
|
92 |
provenance.setProvisionMode(it.get("provisionMode").getAsString()); |
|
93 |
provenance.setDatasource(it.get("datasource").getAsString()); |
|
94 |
provenance.setPublisher(getStringValue(it, "publisher")); |
|
95 |
return provenance; |
|
96 |
}) |
|
97 |
.collect(Collectors.toList()) |
|
98 |
); |
|
89 |
dli.setDatasourceProvenance(A); |
|
90 |
dli.setCompletionStatus(CompletionStatus.complete.toString()); |
|
99 | 91 |
return dli; |
100 | 92 |
} |
101 | 93 |
|
94 |
private SubjectType createSubjectType(final JsonObject item){ |
|
95 |
if (item.has("term") && !item.get("term").isJsonNull()) |
|
96 |
return new SubjectType(item.get("scheme").getAsString(), item.get("term").getAsString()); |
|
97 |
else { |
|
98 |
return null; |
|
99 |
} |
|
100 |
} |
|
101 |
|
|
102 | 102 |
private List<String> getStringValues(final JsonObject root, final String key) { |
103 | 103 |
|
104 | 104 |
if (root.has(key) && root.get(key).isJsonArray()) { |
... | ... | |
127 | 127 |
|
128 | 128 |
} |
129 | 129 |
|
130 |
|
|
130 |
@Required |
|
131 | 131 |
public void setMongoClient(final MongoClient client) { |
132 | 132 |
this.client = client; |
133 | 133 |
} |
modules/dnet-dli/trunk/src/main/java/eu/dnetlib/dli/collector/plugin/CrossRefIterator.java | ||
---|---|---|
64 | 64 |
} |
65 | 65 |
if (buffer.isEmpty()) return null; |
66 | 66 |
final JsonObject item = buffer.remove(0).getAsJsonObject(); |
67 |
final DLIResolvedObject object = new DLIResolvedObject();
|
|
68 |
final String id = item.get("Source").getAsJsonObject().get("Identifier").getAsJsonObject().get("ID").getAsString();
|
|
69 |
final String id_type = item.get("Source").getAsJsonObject().get("Identifier").getAsJsonObject().get("IDScheme").getAsString();
|
|
70 |
object.setPid(id);
|
|
71 |
object.setPidType(id_type);
|
|
72 |
final String type = item.get("Source").getAsJsonObject().get("Type").getAsJsonObject().get("Name").getAsString();
|
|
73 |
object.setType(type.equals("literature")? ObjectType.publication:ObjectType.dataset);
|
|
74 |
final String relType = item.get("RelationshipType").getAsJsonObject().get("Name").getAsString();
|
|
75 |
object.setDatasourceProvenance(Collections.singletonList(new DLIObjectProvenance().setDatasource("Crossref").setDatasourceId("dli_________::crossref")));
|
|
76 |
DLIObjectRelation relation = new DLIObjectRelation();
|
|
77 |
relation.setRelationSemantics(relType);
|
|
78 |
final String t_id = item.get("Target").getAsJsonObject().get("Identifier").getAsJsonObject().get("ID").getAsString();
|
|
79 |
final String t_id_type = item.get("Target").getAsJsonObject().get("Identifier").getAsJsonObject().get("IDScheme").getAsString();
|
|
80 |
final String t_type = item.get("Target").getAsJsonObject().get("Type").getAsJsonObject().get("Name").getAsString();
|
|
67 |
try {
|
|
68 |
final DLIResolvedObject object = new DLIResolvedObject();
|
|
69 |
final String id = item.get("Source").getAsJsonObject().get("Identifier").getAsJsonObject().get("ID").getAsString();
|
|
70 |
final String id_type = item.get("Source").getAsJsonObject().get("Identifier").getAsJsonObject().get("IDScheme").getAsString();
|
|
71 |
object.setPid(id);
|
|
72 |
object.setPidType(id_type);
|
|
73 |
final String type = item.get("Source").getAsJsonObject().get("Type").getAsJsonObject().get("Name").isJsonNull()?"unknownw":item.get("Source").getAsJsonObject().get("Type").getAsJsonObject().get("Name").getAsString();
|
|
74 |
object.setType(type.equals("literature") ? ObjectType.publication : ObjectType.dataset);
|
|
75 |
final String relType = item.get("RelationshipType").getAsJsonObject().get("Name").getAsString();
|
|
76 |
object.setDatasourceProvenance(Collections.singletonList(new DLIObjectProvenance().setDatasource("Crossref").setDatasourceId("dli_________::crossref")));
|
|
77 |
DLIObjectRelation relation = new DLIObjectRelation();
|
|
78 |
relation.setRelationSemantics(relType);
|
|
79 |
final String t_id = item.get("Target").getAsJsonObject().get("Identifier").getAsJsonObject().get("ID").getAsString();
|
|
80 |
final String t_id_type = item.get("Target").getAsJsonObject().get("Identifier").getAsJsonObject().get("IDScheme").getAsString();
|
|
81 | 81 |
|
82 |
relation.setTargetPID(new PID(t_id,t_id_type)); |
|
83 |
relation.setTargetType(t_type.equals("literature")? ObjectType.publication:ObjectType.dataset); |
|
84 |
object.setRelations(Collections.singletonList(relation)); |
|
85 |
relation.setCompletionStatus("incomplete"); |
|
86 |
relation.setRelationProvenance(Collections.singletonList(new DLIObjectProvenance().setDatasource("Crossref").setDatasourceId("dli_________::crossref"))); |
|
87 |
try { |
|
88 |
StringWriter writer = new StringWriter(16); |
|
89 |
NoIndentWriter out = new NoIndentWriter(writer); |
|
90 |
resolverSerializer.removeAttribute("object"); |
|
91 |
resolverSerializer.setAttribute("object", object); |
|
92 |
resolverSerializer.write(out); |
|
93 |
String result = writer.toString(); |
|
94 |
writer.close(); |
|
95 |
return result; |
|
96 |
} catch (Throwable e) { |
|
82 |
final String t_type = item.get("Target").getAsJsonObject().get("Type").getAsJsonObject().get("Name").isJsonNull()?"unknown":item.get("Target").getAsJsonObject().get("Type").getAsJsonObject().get("Name").getAsString(); |
|
83 |
|
|
84 |
relation.setTargetPID(new PID(t_id, t_id_type)); |
|
85 |
relation.setTargetType(t_type.equals("literature") ? ObjectType.publication : ObjectType.dataset); |
|
86 |
object.setRelations(Collections.singletonList(relation)); |
|
87 |
relation.setCompletionStatus("incomplete"); |
|
88 |
relation.setRelationProvenance(Collections.singletonList(new DLIObjectProvenance().setDatasource("Crossref").setDatasourceId("dli_________::crossref"))); |
|
89 |
try { |
|
90 |
StringWriter writer = new StringWriter(16); |
|
91 |
NoIndentWriter out = new NoIndentWriter(writer); |
|
92 |
resolverSerializer.removeAttribute("object"); |
|
93 |
resolverSerializer.setAttribute("object", object); |
|
94 |
resolverSerializer.write(out); |
|
95 |
String result = writer.toString(); |
|
96 |
writer.close(); |
|
97 |
return result; |
|
98 |
} catch (Throwable e) { |
|
99 |
throw new RuntimeException(e); |
|
100 |
} |
|
101 |
}catch (Throwable e) { |
|
102 |
System.out.println(item.toString()); |
|
97 | 103 |
throw new RuntimeException(e); |
98 | 104 |
} |
99 | 105 |
} |
... | ... | |
108 | 114 |
if (jElement.getAsJsonObject().has("message")) |
109 | 115 |
{ |
110 | 116 |
JsonObject message = jElement.getAsJsonObject().get("message").getAsJsonObject(); |
111 |
nextCursor = message.get("next-cursor").getAsString(); |
|
117 |
if (message.has("next-cursor") && !message.get("next-cursor").isJsonNull()) |
|
118 |
nextCursor = message.get("next-cursor").getAsString(); |
|
119 |
else |
|
120 |
nextCursor = null; |
|
112 | 121 |
total = message.get("total-results").getAsInt(); |
113 | 122 |
final JsonArray items = message.get("link-packages").getAsJsonArray(); |
114 | 123 |
items.forEach(buffer::add); |
115 |
return true;
|
|
124 |
return nextCursor!=null;
|
|
116 | 125 |
} |
117 | 126 |
} |
118 | 127 |
return false; |
modules/dnet-dli/trunk/src/main/resources/eu/dnetlib/dli/resolver/applicationContext-dli-resolver.xml | ||
---|---|---|
46 | 46 |
<!--<property name="cache" ref="dliResolverCache"/>--> |
47 | 47 |
<!--</bean>--> |
48 | 48 |
|
49 |
<bean id="dataciteOfflineResolver" class="eu.dnetlib.dli.resolver.DataciteOfflineResolver" p:order="0" p:availableOffline="true"> |
|
49 |
<bean id="dataciteOfflineResolver" |
|
50 |
class="eu.dnetlib.dli.resolver.DataciteOfflineResolver" |
|
51 |
p:order="0" p:availableOffline="true" |
|
52 |
p:mongoClient-ref="mongoClient"> |
|
50 | 53 |
<property name="cache" ref="dliResolverCache"/> |
54 |
|
|
51 | 55 |
</bean> |
52 | 56 |
|
53 | 57 |
<bean id="ncbinResolver" class="eu.dnetlib.dli.resolver.NCBINResolver" p:order="4" p:availableOffline="false"> |
modules/dnet-dli/trunk/src/main/resources/eu/dnetlib/dli/templates/Scholixv1.st | ||
---|---|---|
19 | 19 |
}$ |
20 | 20 |
<source> |
21 | 21 |
<identifier> |
22 |
<identifier>$object.pid$</identifier>
|
|
23 |
<schema>$object.type$</schema>
|
|
22 |
<identifier>$object.escapedXMLPid$</identifier>
|
|
23 |
<schema>$object.pidType$</schema>
|
|
24 | 24 |
</identifier> |
25 | 25 |
<objectType> |
26 | 26 |
<type>$object.type$</type> |
... | ... | |
29 | 29 |
<target> |
30 | 30 |
$object.relations:{ |
31 | 31 |
<identifier> |
32 |
<identifier>$it.targetPID.id$</identifier>
|
|
32 |
<identifier>$it.targetPID.escapeXMLId$</identifier>
|
|
33 | 33 |
<schema>$it.targetPID.type$</schema> |
34 | 34 |
</identifier> |
35 | 35 |
<objectType> |
modules/dnet-dli/trunk/pom.xml | ||
---|---|---|
10 | 10 |
<modelVersion>4.0.0</modelVersion> |
11 | 11 |
<groupId>eu.dnetlib</groupId> |
12 | 12 |
<artifactId>dnet-dli</artifactId> |
13 |
<packaging>pom</packaging>
|
|
13 |
<packaging>jar</packaging>
|
|
14 | 14 |
<version>1.0.0-SNAPSHOT</version> |
15 |
|
|
16 |
|
|
15 | 17 |
<scm> |
16 | 18 |
<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet50/modules/dnet-dli/trunk</developerConnection> |
17 | 19 |
</scm> |
Also available in: Unified diff
added Crossref Event Data