Revision 49676
Added by Sandro La Bruzzo over 6 years ago
modules/dnet-dli/trunk/src/test/java/eu/dnetlib/resolver/DLIResolverTest.java | ||
---|---|---|
117 | 117 |
|
118 | 118 |
@Test |
119 | 119 |
public void andsResolverTest() throws Exception { |
120 |
andsDataciteResolver.retrievePID("https://researchdata.ands.org.au/registry/registry_object/view/2756", "url"); |
|
121 |
andsDataciteResolver.retrievePID("https://researchdata.ands.org.au/registry/registry_object/view/2756", "url"); |
|
122 |
andsDataciteResolver.retrievePID("https://researchdata.ands.org.au/registry/registry_object/view/2756", "url"); |
|
123 |
final DLIResolvedObject doi = (DLIResolvedObject) andsDataciteResolver.retrievePID("https://researchdata.ands.org.au/registry/registry_object/view/2756", "url"); |
|
120 |
andsDataciteResolver.retrievePID("https://researchdata.ands.org.au/registry/registry_object/view/2756", "url",false);
|
|
121 |
andsDataciteResolver.retrievePID("https://researchdata.ands.org.au/registry/registry_object/view/2756", "url",false);
|
|
122 |
andsDataciteResolver.retrievePID("https://researchdata.ands.org.au/registry/registry_object/view/2756", "url",false);
|
|
123 |
final DLIResolvedObject doi = (DLIResolvedObject) andsDataciteResolver.retrievePID("https://researchdata.ands.org.au/registry/registry_object/view/2756", "url",false);
|
|
124 | 124 |
Assert.assertNotNull(doi); |
125 | 125 |
Assert.assertNotNull(doi.getTitles()); |
126 | 126 |
Assert.assertNotNull(doi.getDatasourceProvenance()); |
... | ... | |
130 | 130 |
|
131 | 131 |
@Test |
132 | 132 |
public void pubmedResolverTest() throws Exception { |
133 |
final DLIResolvedObject pubmedObject = (DLIResolvedObject) pubMedResolver.retrievePID("355643", "pubmedid"); |
|
133 |
final DLIResolvedObject pubmedObject = (DLIResolvedObject) pubMedResolver.retrievePID("355643", "pubmedid",false);
|
|
134 | 134 |
Assert.assertNotNull(pubmedObject); |
135 | 135 |
Assert.assertNotNull(pubmedObject.getAuthors()); |
136 | 136 |
Assert.assertNotNull(pubmedObject.getTitles()); |
... | ... | |
138 | 138 |
|
139 | 139 |
@Test |
140 | 140 |
public void openaireResolverTest() throws Exception { |
141 |
final DLIResolvedObject pubmedObject = (DLIResolvedObject) openaireResolver.retrievePID("oai:dnet:od_______908::f602032c963cc56d6c9e80508ece44d3", "openaire"); |
|
141 |
final DLIResolvedObject pubmedObject = (DLIResolvedObject) openaireResolver.retrievePID("oai:dnet:od_______908::f602032c963cc56d6c9e80508ece44d3", "openaire",false);
|
|
142 | 142 |
System.out.println("pubmedObject = " + pubmedObject); |
143 | 143 |
Assert.assertNotNull(pubmedObject); |
144 | 144 |
Assert.assertNotNull(pubmedObject.getAuthors()); |
... | ... | |
147 | 147 |
|
148 | 148 |
@Test |
149 | 149 |
public void crossRefResolverTest() throws Exception { |
150 |
final DLIResolvedObject pubmedObject = (DLIResolvedObject) crossrefResolver.retrievePID("10.1109/5.771073", "Handle"); |
|
150 |
final DLIResolvedObject pubmedObject = (DLIResolvedObject) crossrefResolver.retrievePID("10.1109/5.771073", "Handle",false);
|
|
151 | 151 |
Assert.assertNotNull(pubmedObject); |
152 | 152 |
Assert.assertNotNull(pubmedObject.getAuthors()); |
153 | 153 |
Assert.assertNotNull(pubmedObject.getTitles()); |
... | ... | |
155 | 155 |
|
156 | 156 |
@Test |
157 | 157 |
public void rcsbResolverTest() throws Exception { |
158 |
final DLIResolvedObject pubmedObject = (DLIResolvedObject) rcsbResolver.retrievePID("2Y9C", "pdb"); |
|
158 |
final DLIResolvedObject pubmedObject = (DLIResolvedObject) rcsbResolver.retrievePID("2Y9C", "pdb",false);
|
|
159 | 159 |
Assert.assertNotNull(pubmedObject); |
160 | 160 |
Assert.assertNotNull(pubmedObject.getAuthors()); |
161 | 161 |
Assert.assertNotNull(pubmedObject.getTitles()); |
... | ... | |
164 | 164 |
|
165 | 165 |
@Test |
166 | 166 |
public void enaResolverTest() throws java.lang.Exception { |
167 |
final DLIResolvedObject enaObject = (DLIResolvedObject) enaResolver.retrievePID("KP866224", "ena"); |
|
167 |
final DLIResolvedObject enaObject = (DLIResolvedObject) enaResolver.retrievePID("KP866224", "ena",false);
|
|
168 | 168 |
Assert.assertNotNull(enaObject); |
169 | 169 |
Assert.assertNotNull(enaObject.getAuthors()); |
170 | 170 |
Assert.assertNotNull(enaObject.getTitles()); |
... | ... | |
180 | 180 |
"10.6085/AA/TPT006_061MXTI061R00_20020806.50.3", "10013/epic.26895.d001" }; |
181 | 181 |
int k = 0; |
182 | 182 |
double average_sum = 0.0f; |
183 |
DLIResolvedObject object = (DLIResolvedObject) dataciteResolver.retrievePID(dois[0], "handle"); |
|
183 |
DLIResolvedObject object = (DLIResolvedObject) dataciteResolver.retrievePID(dois[0], "handle",false);
|
|
184 | 184 |
Assert.assertNotNull(object); |
185 |
object = (DLIResolvedObject) dataciteResolver.retrievePID(dois[1], "doi"); |
|
185 |
object = (DLIResolvedObject) dataciteResolver.retrievePID(dois[1], "doi",false);
|
|
186 | 186 |
Assert.assertNotNull(object); |
187 | 187 |
log.info(object); |
188 | 188 |
log.info(object.getSubjects()); |
modules/dnet-dli/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/ResolveAndIndexJobNode.java | ||
---|---|---|
193 | 193 |
|
194 | 194 |
private DLIResolvedObject resolveRelation(final PID currentPid, final ObjectProvenance provenance) { |
195 | 195 |
for (PIDResolver resolver : pluginResolver) { |
196 |
final DLIResolvedObject currentIdentifier = (DLIResolvedObject) resolver.retrievePID(currentPid.getId(), currentPid.getType()); |
|
196 |
final DLIResolvedObject currentIdentifier = (DLIResolvedObject) resolver.retrievePID(currentPid.getId(), currentPid.getType(), false);
|
|
197 | 197 |
|
198 | 198 |
if (currentIdentifier != null && |
199 | 199 |
!StringUtils.isBlank(currentIdentifier.getPid()) && |
modules/dnet-dli/trunk/src/main/java/eu/dnetlib/resolver/mdstore/plugin/DLIRecordResolverFactory.java | ||
---|---|---|
12 | 12 |
|
13 | 13 |
public class DLIRecordResolverFactory implements RecordResolverFactory { |
14 | 14 |
@Override |
15 |
public RecordResolver createResolver(BlockingQueue<DBObject> queue, MongoCollection<DBObject> outputCollection, ResolverSerializer serializer, List<PIDResolver> pluginResolver) { |
|
15 |
public RecordResolver createResolver(BlockingQueue<DBObject> queue, MongoCollection<DBObject> outputCollection, ResolverSerializer serializer, List<PIDResolver> pluginResolver, boolean offline) {
|
|
16 | 16 |
|
17 |
return createResolver(System.currentTimeMillis(), queue, outputCollection, serializer, pluginResolver); |
|
17 |
return createResolver(System.currentTimeMillis(), queue, outputCollection, serializer, pluginResolver, offline);
|
|
18 | 18 |
} |
19 | 19 |
|
20 | 20 |
@Override |
21 |
public RecordResolver createResolver(long identifier, BlockingQueue<DBObject> queue, MongoCollection<DBObject> outputCollection, ResolverSerializer serializer, List<PIDResolver> pluginResolver) { |
|
21 |
public RecordResolver createResolver(long identifier, BlockingQueue<DBObject> queue, MongoCollection<DBObject> outputCollection, ResolverSerializer serializer, List<PIDResolver> pluginResolver, boolean offline) {
|
|
22 | 22 |
final DLIRecordResolver resolver = new DLIRecordResolver(System.currentTimeMillis()); |
23 | 23 |
resolver.setInputQueue(queue); |
24 | 24 |
resolver.setOutputCollection(outputCollection); |
25 | 25 |
resolver.setSerializer(serializer); |
26 | 26 |
resolver.setPluginResolver(pluginResolver); |
27 |
resolver.setOffline(offline); |
|
27 | 28 |
return resolver; |
28 | 29 |
} |
29 | 30 |
} |
modules/dnet-dli/trunk/src/main/java/eu/dnetlib/resolver/mdstore/plugin/DLIRecordResolver.java | ||
---|---|---|
67 | 67 |
log.debug("tryToResolveRelation " + currentPid); |
68 | 68 |
|
69 | 69 |
for (PIDResolver resolver : pluginResolver) { |
70 |
final DLIResolvedObject currentIdentifier = (DLIResolvedObject) resolver.retrievePID(currentPid.getId(), currentPid.getType()); |
|
70 |
final DLIResolvedObject currentIdentifier = (DLIResolvedObject) resolver.retrievePID(currentPid.getId(), currentPid.getType(), offline);
|
|
71 | 71 |
|
72 | 72 |
|
73 | 73 |
if (currentIdentifier != null && !StringUtils.isBlank(currentIdentifier.getPid()) && currentIdentifier.getPid().toLowerCase().equals(currentPid.getId().toLowerCase())) { |
... | ... | |
83 | 83 |
private boolean tryToResolveRecord(final DLIResolvedObject object) { |
84 | 84 |
|
85 | 85 |
for (PIDResolver resolver : pluginResolver) { |
86 |
final DLIResolvedObject resolvedObject = (DLIResolvedObject) resolver.retrievePID(object.getPid(), object.getPidType()); |
|
86 |
final DLIResolvedObject resolvedObject = (DLIResolvedObject) resolver.retrievePID(object.getPid(), object.getPidType(), offline);
|
|
87 | 87 |
if (resolvedObject != null && |
88 | 88 |
resolvedObject.getCompletionStatus() != null && |
89 | 89 |
resolvedObject.getCompletionStatus().equals(CompletionStatus.complete.toString())) { |
modules/dnet-dli/trunk/src/main/java/eu/dnetlib/dli/resolver/CrossrefResolver.java | ||
---|---|---|
35 | 35 |
*/ |
36 | 36 |
private final static String baseUrlCrossref = "http://api.crossref.org/works/"; |
37 | 37 |
|
38 |
|
|
38 | 39 |
/** |
39 | 40 |
* The parser. |
40 | 41 |
*/ |
... | ... | |
92 | 93 |
return iterator.next().toJson(); |
93 | 94 |
} |
94 | 95 |
return null; |
96 |
} |
|
95 | 97 |
|
96 | 98 |
|
97 | 99 |
|
98 | 100 |
|
99 |
} |
|
100 | 101 |
|
101 | 102 |
|
102 | 103 |
|
103 |
|
|
104 |
|
|
105 | 104 |
} |
modules/dnet-dli/trunk/src/main/java/eu/dnetlib/dli/resolver/DLIOfflineResolver.java | ||
---|---|---|
1 |
package eu.dnetlib.dli.resolver; |
|
2 |
|
|
3 |
import com.google.gson.Gson; |
|
4 |
import com.google.gson.GsonBuilder; |
|
5 |
import com.mongodb.DBObject; |
|
6 |
import com.mongodb.MongoClient; |
|
7 |
import com.mongodb.QueryBuilder; |
|
8 |
import com.mongodb.client.FindIterable; |
|
9 |
import com.mongodb.client.MongoCollection; |
|
10 |
import com.mongodb.client.MongoCursor; |
|
11 |
import com.mongodb.client.MongoDatabase; |
|
12 |
import eu.dnetlib.dli.resolver.model.DLIResolvedObject; |
|
13 |
import eu.dnetlib.dli.resolver.model.DLIResolvedObjectDeserializer; |
|
14 |
import eu.dnetlib.pid.resolver.AbstractPIDResolver; |
|
15 |
|
|
16 |
import org.apache.commons.logging.Log; |
|
17 |
import org.apache.commons.logging.LogFactory; |
|
18 |
import org.bson.Document; |
|
19 |
import org.bson.conversions.Bson; |
|
20 |
import org.springframework.beans.factory.annotation.Autowired; |
|
21 |
|
|
22 |
public class DLIOfflineResolver extends AbstractPIDResolver { |
|
23 |
|
|
24 |
/** |
|
25 |
* The Constant log. |
|
26 |
*/ |
|
27 |
private static final Log log = LogFactory.getLog(DLIOfflineResolver.class); |
|
28 |
|
|
29 |
@Autowired |
|
30 |
private MongoClient mongoClient; |
|
31 |
|
|
32 |
private MongoDatabase db; |
|
33 |
|
|
34 |
private Gson g = new GsonBuilder().registerTypeAdapter(DLIResolvedObject.class, new DLIResolvedObjectDeserializer()).create(); |
|
35 |
|
|
36 |
@Override |
|
37 |
protected boolean canResolvePid(final String pidType) { |
|
38 |
return isAvailableOffline(); |
|
39 |
} |
|
40 |
|
|
41 |
@Override |
|
42 |
protected DLIResolvedObject resolve(final String pid, final String pidType) { |
|
43 |
|
|
44 |
final String dliItem = retrieveDLIFromDump(pid.toLowerCase().trim(),pidType.toLowerCase().trim()); |
|
45 |
if (dliItem!= null) |
|
46 |
return g.fromJson(dliItem, DLIResolvedObject.class); |
|
47 |
return null; |
|
48 |
|
|
49 |
} |
|
50 |
|
|
51 |
|
|
52 |
private String retrieveDLIFromDump(final String pid, final String pidType) { |
|
53 |
log.debug(String.format("trying to find %s of type %s", pid, pidType)); |
|
54 |
if (db == null) { |
|
55 |
db = mongoClient.getDatabase("dli_resolved"); |
|
56 |
} |
|
57 |
final MongoCollection<Document> crossRef = db.getCollection("dli"); |
|
58 |
DBObject query = QueryBuilder.start("pid").is(pid).and("pid_type").is(pidType).get(); |
|
59 |
FindIterable<Document> documents = crossRef.find((Bson) query).limit(1); |
|
60 |
MongoCursor<Document> iterator = documents.iterator(); |
|
61 |
if (iterator.hasNext()){ |
|
62 |
log.debug("found pid "+pid); |
|
63 |
return iterator.next().get("body").toString(); |
|
64 |
} |
|
65 |
return null; |
|
66 |
} |
|
67 |
|
|
68 |
|
|
69 |
} |
modules/dnet-dli/trunk/src/main/resources/eu/dnetlib/dli/resolver/applicationContext-dli-resolver.xml | ||
---|---|---|
15 | 15 |
p:cacheManager-ref="dliCacheManager" p:cacheName="dliResolverCache"/> |
16 | 16 |
|
17 | 17 |
|
18 |
<bean id="pubmedResolver" class="eu.dnetlib.dli.resolver.PubMedResolver" p:order="0"> |
|
18 |
<bean id="pubmedResolver" class="eu.dnetlib.dli.resolver.PubMedResolver" p:order="0" p:availableOffline="false">
|
|
19 | 19 |
<property name="cache" ref="dliResolverCache"/> |
20 | 20 |
</bean> |
21 | 21 |
|
22 | 22 |
<bean id="rcsbParser" class="eu.dnetlib.dli.resolver.RCSBParser"/> |
23 | 23 |
|
24 |
<bean id="rcsbResolver" class="eu.dnetlib.dli.resolver.RCSBResolver" p:order="1"> |
|
24 |
<bean id="rcsbResolver" class="eu.dnetlib.dli.resolver.RCSBResolver" p:order="1" p:availableOffline="false">
|
|
25 | 25 |
<property name="cache" ref="dliResolverCache"/> |
26 | 26 |
</bean> |
27 | 27 |
|
28 |
<bean id="enaResolver" class="eu.dnetlib.dli.resolver.ENAResolver" p:order="2"> |
|
28 |
<bean id="enaResolver" class="eu.dnetlib.dli.resolver.ENAResolver" p:order="2" p:availableOffline="false">
|
|
29 | 29 |
<property name="cache" ref="dliResolverCache"/> |
30 | 30 |
</bean> |
31 | 31 |
|
32 | 32 |
|
33 |
<bean id="andsResolver" class="eu.dnetlib.dli.resolver.ANDSResolver" p:order="3"> |
|
33 |
<bean id="andsResolver" class="eu.dnetlib.dli.resolver.ANDSResolver" p:order="3" p:availableOffline="false">
|
|
34 | 34 |
<property name="cache" ref="dliResolverCache"/> |
35 | 35 |
</bean> |
36 | 36 |
|
37 |
<bean id="crossrefResolver" class="eu.dnetlib.dli.resolver.CrossrefResolver" p:order="0"> |
|
37 |
<bean id="crossrefResolver" class="eu.dnetlib.dli.resolver.CrossrefResolver" p:order="0" p:availableOffline="true">
|
|
38 | 38 |
<property name="cache" ref="dliResolverCache"/> |
39 | 39 |
</bean> |
40 | 40 |
|
41 |
<bean id="dataciteResolver" class="eu.dnetlib.dli.resolver.DataciteResolver" p:order="0">
|
|
41 |
<bean id="dliOfflineResolver" class="eu.dnetlib.dli.resolver.DLIOfflineResolver" p:order="0" p:availableOffline="true">
|
|
42 | 42 |
<property name="cache" ref="dliResolverCache"/> |
43 | 43 |
</bean> |
44 | 44 |
|
45 |
<bean id="ncbinResolver" class="eu.dnetlib.dli.resolver.NCBINResolver" p:order="0">
|
|
45 |
<bean id="dataciteResolver" class="eu.dnetlib.dli.resolver.DataciteResolver" p:order="0" p:availableOffline="false">
|
|
46 | 46 |
<property name="cache" ref="dliResolverCache"/> |
47 | 47 |
</bean> |
48 | 48 |
|
49 |
<bean id="ncbinParser" class="eu.dnetlib.dli.resolver.NCBINParser"/> |
|
49 |
<bean id="ncbinResolver" class="eu.dnetlib.dli.resolver.NCBINResolver" p:order="0" p:availableOffline="false"> |
|
50 |
<property name="cache" ref="dliResolverCache"/> |
|
51 |
</bean> |
|
50 | 52 |
|
53 |
<bean id="ncbinParser" class="eu.dnetlib.dli.resolver.NCBINParser" /> |
|
54 |
|
|
51 | 55 |
<bean id="crossrefRecordParserJSON" class="eu.dnetlib.dli.resolver.CrossRefParserJSON"/> |
52 | 56 |
|
53 | 57 |
<bean id="openaireResolver" class="eu.dnetlib.dli.resolver.OpenaireResolver" p:order="0" |
54 |
p:cache-ref="dliResolverCache"/> |
|
58 |
p:cache-ref="dliResolverCache" p:availableOffline="false"/>
|
|
55 | 59 |
|
56 | 60 |
|
57 | 61 |
<bean id="dliResolverPlugin" class="eu.dnetlib.pid.resolver.mdstore.plugin.ResolverMDStorePlugin"/> |
modules/dnet-dli/trunk/src/main/resources/eu/dnetlib/dli/workflows/repo-hi/dli_aggregation_wf.xml.st | ||
---|---|---|
14 | 14 |
<WORKFLOW_FAMILY>aggregator</WORKFLOW_FAMILY> |
15 | 15 |
<WORKFLOW_PRIORITY>$priority$</WORKFLOW_PRIORITY> |
16 | 16 |
<DATASOURCE id="$dsId$" interface="$interface$" /> |
17 |
|
|
18 | 17 |
<CONFIGURATION status="WAIT_SYS_SETTINGS" start="MANUAL"> |
19 | 18 |
<PARAMETERS> |
20 | 19 |
<PARAM name="collMdstoreId" description="Store for collected records" required="true" managedBy="system" category="MDSTORE_ID"/> |
... | ... | |
24 | 23 |
<PARAM name="unknownMdstoreId" description="Store for unknown records" required="true" managedBy="system" category="MDSTORE_ID"/> |
25 | 24 |
<PARAM name="cleanTransformationRuleId" description="Transformation Rule Identifier" required="true" managedBy="user" category="TRANSFORMATION_RULE_ID" type="string" function="listProfiles('TransformationRuleDSResourceType', '//TITLE', 'DLI:')"/> |
26 | 25 |
<PARAM description="Type of Transformation" function="validValues(['simpleTransform', 'transformAndUnpack'])" managedBy="user" name="typeOfTransform" required="true" type="string">simpleTransform</PARAM> |
26 |
<PARAM description="Resolving offline" function="validValues(['false', 'true'])" managedBy="user" name="offlineResolving" required="true" type="string">false</PARAM> |
|
27 | 27 |
</PARAMETERS> |
28 | 28 |
<WORKFLOW> |
29 | 29 |
<NODE isStart="true" name="collection" type="LaunchWorkflowTemplate"> |
... | ... | |
72 | 72 |
<ENTRY key="interface" value="$interface$" /> |
73 | 73 |
<ENTRY key="collMdstoreId" ref="collMdstoreId" /> |
74 | 74 |
<ENTRY key="cleanMdstoreId" ref="cleanMdstoreId" /> |
75 |
|
|
75 |
<ENTRY key="offlineResolving" ref="offlineResolving" /> |
|
76 | 76 |
</MAP> |
77 | 77 |
</PARAM> |
78 | 78 |
</PARAMETERS> |
modules/dnet-dli/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/linkprovider/link_provider_resolver_template.xml | ||
---|---|---|
14 | 14 |
<PARAM name="interface" description="Datasource Interface" required="true" type="string"/> |
15 | 15 |
<PARAM name="collMdstoreId" description="Store for collected records" required="true" type="string"/> |
16 | 16 |
<PARAM name="cleanMdstoreId" description="Store for cleaned records" required="true" type="string"/> |
17 |
<PARAM name="numberOfThreads" description="number of threads for resolve PID" required="true" |
|
18 |
type="string" default="4"/>
|
|
17 |
<PARAM name="numberOfThreads" description="number of threads for resolve PID" required="true" type="string" default="4"/>
|
|
18 |
<PARAM name="offlineResolving" description="decide if you want resolve offline using only the resolved item" required="true" type="boolean" default="false"/>
|
|
19 | 19 |
</PARAMETERS> |
20 | 20 |
<WORKFLOW> |
21 | 21 |
|
... | ... | |
25 | 25 |
<PARAM name="mdStoreId" ref="cleanMdstoreId"/> |
26 | 26 |
<PARAM name="pluginName" value="dliResolverPlugin"/> |
27 | 27 |
<PARAM name="numberOfThreads" ref="numberOfThreads"/> |
28 |
|
|
28 |
<PARAM name="offline" ref="offlineResolving"/> |
|
29 | 29 |
</PARAMETERS> |
30 | 30 |
<ARCS> |
31 | 31 |
<ARC to="UPDATE_INFO"/> |
modules/dnet-wds/trunk/test/java/eu/dnetlib/wds/resolver/ResolverTest.java | ||
---|---|---|
46 | 46 |
@Test |
47 | 47 |
public void testDLIResolvePublication() throws Exception { |
48 | 48 |
|
49 |
final ResolvedObject openaire = dliResolver.retrievePID("10.1016/j.quascirev.2011.06.004", "doi"); |
|
49 |
final ResolvedObject openaire = dliResolver.retrievePID("10.1016/j.quascirev.2011.06.004", "doi", false);
|
|
50 | 50 |
|
51 | 51 |
final ResolvedObject doi_stored = store.getRecord("10.1016/j.quascirev.2011.06.004", "doi"); |
52 | 52 |
assertNotNull(doi_stored); |
... | ... | |
60 | 60 |
|
61 | 61 |
@Test |
62 | 62 |
public void testOpenAireResolvePublication() throws Exception { |
63 |
final ResolvedObject openaire = openAireResolver.retrievePID("10.1126/science.1141038", "DOI"); |
|
63 |
final ResolvedObject openaire = openAireResolver.retrievePID("10.1126/science.1141038", "DOI", false);
|
|
64 | 64 |
assertNotNull(openaire); |
65 | 65 |
System.out.println(openaire.toJsonString()); |
66 | 66 |
final ResolvedObject doi_stored = store.getRecord("10.1126/science.1141038", "doi"); |
... | ... | |
73 | 73 |
|
74 | 74 |
@Test |
75 | 75 |
public void testSerializationResolvedPublication() throws Exception { |
76 |
final WDSResolvedObject openaire = (WDSResolvedObject) openAireResolver.retrievePID("10.5281/zenodo.11082", "DOI"); |
|
76 |
final WDSResolvedObject openaire = (WDSResolvedObject) openAireResolver.retrievePID("10.5281/zenodo.11082", "DOI", false);
|
|
77 | 77 |
assertNotNull(openaire); |
78 | 78 |
openaire.addDate("IMPORTANT DATE", "20-09-1982"); |
79 | 79 |
|
modules/dnet-wds/trunk/src/main/java/eu/dnetlib/resolver/mdstore/plugin/WDSRecordResolverFactory.java | ||
---|---|---|
12 | 12 |
|
13 | 13 |
public class WDSRecordResolverFactory implements RecordResolverFactory { |
14 | 14 |
@Override |
15 |
public RecordResolver createResolver(BlockingQueue<DBObject> queue, MongoCollection<DBObject> outputCollection, ResolverSerializer serializer, List<PIDResolver> pluginResolver) { |
|
15 |
public RecordResolver createResolver(BlockingQueue<DBObject> queue, MongoCollection<DBObject> outputCollection, ResolverSerializer serializer, List<PIDResolver> pluginResolver, final boolean offline) {
|
|
16 | 16 |
|
17 |
return createResolver(System.currentTimeMillis(), queue, outputCollection, serializer, pluginResolver); |
|
17 |
return createResolver(System.currentTimeMillis(), queue, outputCollection, serializer, pluginResolver, offline);
|
|
18 | 18 |
} |
19 | 19 |
|
20 | 20 |
@Override |
21 |
public RecordResolver createResolver(long identifier, BlockingQueue<DBObject> queue, MongoCollection<DBObject> outputCollection, ResolverSerializer serializer, List<PIDResolver> pluginResolver) { |
|
21 |
public RecordResolver createResolver(long identifier, BlockingQueue<DBObject> queue, MongoCollection<DBObject> outputCollection, ResolverSerializer serializer, List<PIDResolver> pluginResolver, boolean offline) {
|
|
22 | 22 |
final WDSRecordResolver resolver = new WDSRecordResolver(System.currentTimeMillis()); |
23 | 23 |
resolver.setInputQueue(queue); |
24 | 24 |
resolver.setOutputCollection(outputCollection); |
25 | 25 |
resolver.setSerializer(serializer); |
26 | 26 |
resolver.setPluginResolver(pluginResolver); |
27 |
resolver.setOffline(offline); |
|
27 | 28 |
return resolver; |
28 | 29 |
} |
29 | 30 |
} |
modules/dnet-wds/trunk/src/main/java/eu/dnetlib/resolver/mdstore/plugin/WDSRecordResolver.java | ||
---|---|---|
4 | 4 |
import com.mongodb.client.MongoCollection; |
5 | 5 |
import eu.dnetlib.pid.resolver.PIDResolver; |
6 | 6 |
import eu.dnetlib.pid.resolver.mdstore.plugin.AbstractRecordResolver; |
7 |
import eu.dnetlib.pid.resolver.mdstore.plugin.RecordResolver; |
|
8 |
import eu.dnetlib.pid.resolver.mdstore.plugin.ResolverMDStorePlugin; |
|
9 | 7 |
import eu.dnetlib.pid.resolver.mdstore.plugin.ResolverSerializer; |
10 |
import eu.dnetlib.pid.resolver.model.ObjectRelation; |
|
11 | 8 |
import eu.dnetlib.pid.resolver.model.ObjectType; |
12 | 9 |
import eu.dnetlib.pid.resolver.model.PID; |
13 | 10 |
import eu.dnetlib.wds.parser.WDSParser; |
... | ... | |
16 | 13 |
import eu.dnetlib.wds.utils.WDSUtils; |
17 | 14 |
import org.antlr.stringtemplate.StringTemplate; |
18 | 15 |
import org.apache.commons.lang3.StringUtils; |
19 |
import org.apache.commons.logging.Log; |
|
20 |
import org.apache.commons.logging.LogFactory; |
|
21 |
import org.omg.CORBA.CompletionStatus; |
|
22 | 16 |
|
23 | 17 |
import java.util.HashMap; |
24 | 18 |
import java.util.List; |
... | ... | |
62 | 56 |
log.debug("tryToResolveRelation " + currentPid); |
63 | 57 |
|
64 | 58 |
for (PIDResolver resolver : pluginResolver) { |
65 |
final WDSResolvedObject currentIdentifier = (WDSResolvedObject) resolver.retrievePID(currentPid.getId(), currentPid.getType()); |
|
59 |
final WDSResolvedObject currentIdentifier = (WDSResolvedObject) resolver.retrievePID(currentPid.getId(), currentPid.getType(),offline);
|
|
66 | 60 |
|
67 | 61 |
|
68 | 62 |
if (currentIdentifier != null && !StringUtils.isBlank(currentIdentifier.getPid()) && currentIdentifier.getPid().toLowerCase().equals(currentPid.getId().toLowerCase())) { |
modules/dnet-pid-resolver/trunk/src/main/java/eu/dnetlib/pid/resolver/AbstractPIDResolver.java | ||
---|---|---|
29 | 29 |
protected String name; |
30 | 30 |
protected int order; |
31 | 31 |
|
32 |
protected boolean availableOffline; |
|
33 |
|
|
32 | 34 |
@Autowired |
33 | 35 |
private ResolverStore storeManager; |
34 | 36 |
|
... | ... | |
45 | 47 |
|
46 | 48 |
// public abstract void setDatasourceProvenance(final ObjectProvenance provenance, final String namespacePrefix); |
47 | 49 |
|
48 |
public ResolvedObject retrievePID(final String pid, final String pidType) { |
|
49 |
|
|
50 |
public ResolvedObject retrievePID(final String pid, final String pidType, boolean offline) { |
|
51 |
if (offline && !isAvailableOffline()) |
|
52 |
return null; |
|
50 | 53 |
if (!canResolvePid(pidType)) return null; |
51 | 54 |
final String dnetIdentifier = storeManager.generateDNetIdentifier(pid, pidType); |
52 | 55 |
final Element item = cache.get(dnetIdentifier); |
... | ... | |
70 | 73 |
return null; |
71 | 74 |
} |
72 | 75 |
|
73 |
public String retrieveDnetID(final String pid, final String pidType) { |
|
74 |
|
|
76 |
public String retrieveDnetID(final String pid, final String pidType, boolean offline) { |
|
77 |
if (offline && !isAvailableOffline()) |
|
78 |
return null; |
|
75 | 79 |
if (!canResolvePid(pidType)) return null; |
76 | 80 |
final String dnetIdentifier = storeManager.generateDNetIdentifier(pid, pidType); |
77 | 81 |
final Element item = cache.get(dnetIdentifier); |
... | ... | |
153 | 157 |
public int compareTo(PIDResolver o) { |
154 | 158 |
return Integer.compare(getOrder(), o.getOrder()); |
155 | 159 |
} |
160 |
|
|
161 |
|
|
162 |
public boolean isAvailableOffline() { |
|
163 |
return availableOffline; |
|
164 |
} |
|
165 |
|
|
166 |
@Required |
|
167 |
public AbstractPIDResolver setAvailableOffline(boolean availableOffline) { |
|
168 |
this.availableOffline = availableOffline; |
|
169 |
return this; |
|
170 |
} |
|
156 | 171 |
} |
modules/dnet-pid-resolver/trunk/src/main/java/eu/dnetlib/pid/resolver/mdstore/plugin/AbstractRecordResolver.java | ||
---|---|---|
22 | 22 |
protected BlockingQueue<DBObject> inputQueue; |
23 | 23 |
protected MongoCollection<DBObject> outputCollection; |
24 | 24 |
protected ResolverSerializer serializer; |
25 |
protected boolean offline; |
|
25 | 26 |
|
26 | 27 |
public AbstractRecordResolver(final long ts) { |
27 | 28 |
this.timestamp = ts; |
... | ... | |
51 | 52 |
@Override |
52 | 53 |
public Boolean call() throws Exception { |
53 | 54 |
|
54 |
log.info("START HERE!");
|
|
55 |
log.info("START HERE! Ressolving offline:"+offline);
|
|
55 | 56 |
|
56 | 57 |
DBObject currentObject = inputQueue.take(); |
57 | 58 |
int i = 0; |
... | ... | |
92 | 93 |
} |
93 | 94 |
|
94 | 95 |
protected abstract Map<String, ObjectType> tryToResolveRelation(PID targetPID); |
96 |
|
|
97 |
public void setOffline(boolean offline) { |
|
98 |
this.offline = offline; |
|
99 |
} |
|
95 | 100 |
} |
modules/dnet-pid-resolver/trunk/src/main/java/eu/dnetlib/pid/resolver/mdstore/plugin/RecordResolverFactory.java | ||
---|---|---|
9 | 9 |
|
10 | 10 |
public interface RecordResolverFactory { |
11 | 11 |
|
12 |
RecordResolver createResolver(BlockingQueue<DBObject> queue, MongoCollection<DBObject> outputCollection, ResolverSerializer serializer, List<PIDResolver> pluginResolver); |
|
12 |
RecordResolver createResolver(BlockingQueue<DBObject> queue, MongoCollection<DBObject> outputCollection, ResolverSerializer serializer, List<PIDResolver> pluginResolver, final boolean offline);
|
|
13 | 13 |
|
14 |
RecordResolver createResolver(long identifier, BlockingQueue<DBObject> queue, MongoCollection<DBObject> outputCollection, ResolverSerializer serializer, List<PIDResolver> pluginResolver); |
|
14 |
RecordResolver createResolver(long identifier, BlockingQueue<DBObject> queue, MongoCollection<DBObject> outputCollection, ResolverSerializer serializer, List<PIDResolver> pluginResolver, final boolean offline);
|
|
15 | 15 |
|
16 | 16 |
|
17 | 17 |
} |
modules/dnet-pid-resolver/trunk/src/main/java/eu/dnetlib/pid/resolver/mdstore/plugin/ResolverMDStorePlugin.java | ||
---|---|---|
24 | 24 |
import java.util.ArrayList; |
25 | 25 |
import java.util.List; |
26 | 26 |
import java.util.Map; |
27 |
import java.util.Objects; |
|
27 | 28 |
import java.util.concurrent.*; |
28 | 29 |
|
29 | 30 |
import static com.mongodb.client.model.Filters.eq; |
... | ... | |
65 | 66 |
try { |
66 | 67 |
|
67 | 68 |
final String id = params.get("mdStoreId"); |
69 |
final boolean offline = "true".equals(params.get("offline")); |
|
68 | 70 |
|
69 | 71 |
int numberOfThreads = 4; |
70 | 72 |
|
... | ... | |
117 | 119 |
final long ts = System.currentTimeMillis(); |
118 | 120 |
|
119 | 121 |
for (int i = 0; i < numberOfThreads; i++) { |
120 |
final RecordResolver resolver = recordResolverFactory.createResolver(ts, queue, resolvedRecord, resolverSerializer, pluginResolver); |
|
122 |
final RecordResolver resolver = recordResolverFactory.createResolver(ts, queue, resolvedRecord, resolverSerializer, pluginResolver, offline);
|
|
121 | 123 |
responses.add(executor.submit(resolver)); |
122 | 124 |
} |
123 | 125 |
|
modules/dnet-pid-resolver/trunk/src/main/java/eu/dnetlib/pid/resolver/PIDResolver.java | ||
---|---|---|
18 | 18 |
* @param pidType the type of persistent identifier to resolve |
19 | 19 |
* @return the DLIObject resolved if exists, null otherwise |
20 | 20 |
*/ |
21 |
ResolvedObject retrievePID(final String pid, final String pidType); |
|
21 |
ResolvedObject retrievePID(final String pid, final String pidType, final boolean offline);
|
|
22 | 22 |
|
23 | 23 |
/** |
24 | 24 |
* Resolve and return the Dnet Identifier |
... | ... | |
28 | 28 |
* @return the Dnet Identifier of the DLI Object resolved if exists, null otherwise |
29 | 29 |
*/ |
30 | 30 |
|
31 |
String retrieveDnetID(final String pid, final String pidType); |
|
31 |
String retrieveDnetID(final String pid, final String pidType, final boolean offline);
|
|
32 | 32 |
|
33 | 33 |
} |
modules/dnet-pid-resolver/trunk/src/main/java/eu/dnetlib/pid/workflows/nodes/ExecuteMDStorePluginJobNode.java | ||
---|---|---|
18 | 18 |
|
19 | 19 |
private String numberOfThreads; |
20 | 20 |
|
21 |
private boolean offline; |
|
22 |
|
|
21 | 23 |
public String getPluginName() { |
22 | 24 |
return this.pluginName; |
23 | 25 |
} |
... | ... | |
46 | 48 |
job.getParameters().put("mdStoreId", getMdStoreId()); |
47 | 49 |
if (!StringUtils.isEmpty(getNumberOfThreads())) |
48 | 50 |
job.getParameters().put("numberOfThreads", getNumberOfThreads()); |
51 |
job.getParameters().put("offline", ""+isOffline()); |
|
49 | 52 |
} |
50 | 53 |
|
51 | 54 |
public String getNumberOfThreads() { |
... | ... | |
57 | 60 |
} |
58 | 61 |
|
59 | 62 |
|
63 |
public boolean isOffline() { |
|
64 |
return offline; |
|
65 |
} |
|
66 |
|
|
67 |
public ExecuteMDStorePluginJobNode setOffline(boolean offline) { |
|
68 |
this.offline = offline; |
|
69 |
return this; |
|
70 |
} |
|
60 | 71 |
} |
Also available in: Unified diff
Changed Crossref to use dump on mongoDB