Revision 47423
Added by Michele Artini over 7 years ago
modules/dnet-openaireplus-workflows/trunk/src/test/java/eu/dnetlib/msro/openaireplus/api/OpenAIRESubmitterUtilsTest.java | ||
---|---|---|
1 |
package eu.dnetlib.msro.openaireplus.api; |
|
2 |
|
|
3 |
import java.util.Map; |
|
4 |
import java.util.Map.Entry; |
|
5 |
|
|
6 |
import org.junit.Test; |
|
7 |
|
|
8 |
/** |
|
9 |
* Created by Alessia Bardi on 26/05/2017. |
|
10 |
* |
|
11 |
* @author Alessia Bardi |
|
12 |
*/ |
|
13 |
public class OpenAIRESubmitterUtilsTest { |
|
14 |
|
|
15 |
private OpenAIRESubmitterUtils utils = new OpenAIRESubmitterUtils(); |
|
16 |
|
|
17 |
final String fullProject = "info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"; |
|
18 |
final String minimalProject = "info:eu-repo/grantAgreement/EC/FP7/244909///WorkAble"; |
|
19 |
final String onlyTitle = "info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work"; |
|
20 |
|
|
21 |
@Test |
|
22 |
public void testCalculateProjectInfoFull() { |
|
23 |
final Map<String, String> project = utils.calculateProjectInfo(fullProject); |
|
24 |
print(project); |
|
25 |
} |
|
26 |
|
|
27 |
@Test |
|
28 |
public void testCalculateProjectInfoMinimalAcro() { |
|
29 |
final Map<String, String> project = utils.calculateProjectInfo(minimalProject); |
|
30 |
print(project); |
|
31 |
} |
|
32 |
|
|
33 |
@Test |
|
34 |
public void testCalculateProjectInfoOnlyTitle() { |
|
35 |
final Map<String, String> project = utils.calculateProjectInfo(onlyTitle); |
|
36 |
print(project); |
|
37 |
} |
|
38 |
|
|
39 |
private void print(final Map<String, String> map) { |
|
40 |
for (final Entry e : map.entrySet()) { |
|
41 |
System.out.println(e.getKey() + " = " + e.getValue()); |
|
42 |
} |
|
43 |
} |
|
44 |
} |
modules/dnet-openaireplus-workflows/trunk/src/test/java/eu/dnetlib/msro/openaireplus/api/objects/ResultEntryTest.java | ||
---|---|---|
1 |
package eu.dnetlib.msro.openaireplus.api.objects; |
|
2 |
|
|
3 |
import static org.mockito.Matchers.anyString; |
|
4 |
import static org.mockito.Mockito.when; |
|
5 |
|
|
6 |
import java.io.InputStreamReader; |
|
7 |
import java.io.StringReader; |
|
8 |
import java.util.ArrayList; |
|
9 |
import java.util.Arrays; |
|
10 |
import java.util.Properties; |
|
11 |
|
|
12 |
import org.apache.velocity.app.VelocityEngine; |
|
13 |
import org.dom4j.Document; |
|
14 |
import org.dom4j.io.OutputFormat; |
|
15 |
import org.dom4j.io.SAXReader; |
|
16 |
import org.dom4j.io.XMLWriter; |
|
17 |
import org.junit.Before; |
|
18 |
import org.junit.Test; |
|
19 |
import org.junit.runner.RunWith; |
|
20 |
import org.mockito.Mock; |
|
21 |
import org.mockito.invocation.InvocationOnMock; |
|
22 |
import org.mockito.runners.MockitoJUnitRunner; |
|
23 |
import org.mockito.stubbing.Answer; |
|
24 |
|
|
25 |
import com.google.gson.Gson; |
|
26 |
|
|
27 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; |
|
28 |
|
|
29 |
/** |
|
30 |
* Created by michele on 14/12/15. |
|
31 |
*/ |
|
32 |
@RunWith(MockitoJUnitRunner.class) |
|
33 |
public class ResultEntryTest { |
|
34 |
|
|
35 |
private VelocityEngine ve; |
|
36 |
|
|
37 |
@Mock |
|
38 |
private ISLookUpService lookUpService; |
|
39 |
|
|
40 |
@Before |
|
41 |
public void setUp() throws Exception { |
|
42 |
|
|
43 |
final Properties props = new Properties(); |
|
44 |
props.setProperty("resource.loader", "class"); |
|
45 |
props.setProperty("class.resource.loader.class", "org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader"); |
|
46 |
props.setProperty("runtime.log.logsystem.class", "org.apache.velocity.runtime.log.Log4JLogChute"); |
|
47 |
ve = new VelocityEngine(); |
|
48 |
ve.init(props); |
|
49 |
|
|
50 |
when(lookUpService.quickSearchProfile(anyString())).thenAnswer(new Answer<Object>() { |
|
51 |
|
|
52 |
@Override |
|
53 |
public Object answer(final InvocationOnMock invocation) throws Throwable { |
|
54 |
final String query = invocation.getArguments()[0].toString(); |
|
55 |
if (query.contains("dnet:result_typologies")) { |
|
56 |
return Arrays.asList("publication @@@ publication", "dataset @@@ dataset"); |
|
57 |
} else if (query.contains("dnet:access_modes")) { |
|
58 |
return Arrays.asList("OPEN @@@ Open Access"); |
|
59 |
} else if (query.contains("dnet:publication_resource")) { |
|
60 |
return Arrays.asList("0001 @@@ Journal Article"); |
|
61 |
} else if (query.contains("dnet:pid_types")) { |
|
62 |
return Arrays.asList("oai @@@ Open Archive Initiative", "doi @@@ Digital object identifier"); |
|
63 |
} else if (query.contains("dnet:languages")) { |
|
64 |
return Arrays.asList("ita @@@ Italian"); |
|
65 |
} else if (query.contains("ContextDSResourceType")) { |
|
66 |
return Arrays.asList( |
|
67 |
"egi::classification::natsc::math::stats @@@ Statistics and Probability", |
|
68 |
"egi::classification::natsc::math::pure @@@ Pure Mathematics", |
|
69 |
"egi::classification::natsc::math @@@ Mathematics", |
|
70 |
"egi::classification::natsc @@@ Natural Sciences", |
|
71 |
"egi::classification @@@ EGI classification scheme", |
|
72 |
"egi @@@ EGI"); |
|
73 |
} else { |
|
74 |
return new ArrayList<String>(); |
|
75 |
} |
|
76 |
} |
|
77 |
}); |
|
78 |
|
|
79 |
when(lookUpService.getResourceProfileByQuery(anyString())).thenAnswer(new Answer<Object>() { |
|
80 |
|
|
81 |
@Override |
|
82 |
public Object answer(final InvocationOnMock invocation) throws Throwable { |
|
83 |
return "REPO NAME @@@ repo________"; |
|
84 |
} |
|
85 |
}); |
|
86 |
} |
|
87 |
|
|
88 |
@Test |
|
89 |
public void testAsIndexRecord() throws Exception { |
|
90 |
final ResultEntry pub = new ResultEntry(); |
|
91 |
pub.setOriginalId("ORIG_ID_1234"); |
|
92 |
pub.setTitle("TEST TITLE <test>"); |
|
93 |
pub.getAuthors().add("Michele Artini"); |
|
94 |
pub.getAuthors().add("Claudio Atzori"); |
|
95 |
pub.getAuthors().add("Alessia Bardi"); |
|
96 |
pub.setPublisher("Test publisher"); |
|
97 |
pub.setDescription("DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION "); |
|
98 |
pub.setLanguage("ita"); |
|
99 |
pub.setLicenseCode("OPEN"); |
|
100 |
pub.setResourceType("0001"); |
|
101 |
pub.setUrl("http://test.it/a=1&b=2"); |
|
102 |
pub.getPids().add(new PidEntry("doi", "10.000/xyz-123")); |
|
103 |
pub.getPids().add(new PidEntry("oai", "oai:1234")); |
|
104 |
pub.setCollectedFromId("test________::zenodo"); |
|
105 |
pub.setHostedById("test________::UNKNOWN"); |
|
106 |
pub.getLinksToProjects().add("info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus"); |
|
107 |
pub.getLinksToProjects().add("info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"); |
|
108 |
pub.getContexts().add("egi::classification::natsc::math::pure"); |
|
109 |
pub.getContexts().add("egi::classification::natsc::math::stats"); |
|
110 |
final String xml = pub.asOafRecord(ve, lookUpService, "http://oaf/oaf.xsd"); |
|
111 |
|
|
112 |
final Document doc = (new SAXReader()).read(new StringReader(xml)); |
|
113 |
|
|
114 |
final OutputFormat format = OutputFormat.createPrettyPrint(); |
|
115 |
|
|
116 |
final XMLWriter writer = new XMLWriter(System.out, format); |
|
117 |
|
|
118 |
writer.write(doc); |
|
119 |
|
|
120 |
/* writer.close(); */ |
|
121 |
|
|
122 |
} |
|
123 |
|
|
124 |
@Test |
|
125 |
public void testAsIndexRecord_json() throws Exception { |
|
126 |
final ResultEntry pub = (new Gson()).fromJson(new InputStreamReader(getClass().getResourceAsStream("test_record.json")), ResultEntry.class); |
|
127 |
|
|
128 |
final String xml = pub.asOafRecord(ve, lookUpService, "http://oaf/oaf.xsd"); |
|
129 |
|
|
130 |
final Document doc = (new SAXReader()).read(new StringReader(xml)); |
|
131 |
|
|
132 |
final OutputFormat format = OutputFormat.createPrettyPrint(); |
|
133 |
|
|
134 |
final XMLWriter writer = new XMLWriter(System.out, format); |
|
135 |
|
|
136 |
writer.write(doc); |
|
137 |
|
|
138 |
/* writer.close(); */ |
|
139 |
|
|
140 |
} |
|
141 |
|
|
142 |
@Test |
|
143 |
public void testAsIndexRecord_json_with_greek_chars() throws Exception { |
|
144 |
|
|
145 |
final ResultEntry pub = |
|
146 |
(new Gson()).fromJson(new InputStreamReader(getClass().getResourceAsStream("test_record_with_greek_chars.json")), ResultEntry.class); |
|
147 |
|
|
148 |
final String xml = pub.asOafRecord(ve, lookUpService, "http://oaf/oaf.xsd"); |
|
149 |
|
|
150 |
final Document doc = (new SAXReader()).read(new StringReader(xml)); |
|
151 |
|
|
152 |
final OutputFormat format = OutputFormat.createPrettyPrint(); |
|
153 |
|
|
154 |
final XMLWriter writer = new XMLWriter(System.out, format); |
|
155 |
|
|
156 |
writer.write(doc); |
|
157 |
|
|
158 |
/* writer.close(); */ |
|
159 |
|
|
160 |
} |
|
161 |
|
|
162 |
} |
modules/dnet-openaireplus-workflows/trunk/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record.json | ||
---|---|---|
1 |
{ |
|
2 |
"originalId": "ORIG_ID_12345", |
|
3 |
"title": "TEST TITLE", |
|
4 |
"authors": [ |
|
5 |
"Michele Artini", |
|
6 |
"Claudio Atzori", |
|
7 |
"Alessia Bardi" |
|
8 |
], |
|
9 |
"publisher": "Test publisher", |
|
10 |
"description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION", |
|
11 |
"language": "ita", |
|
12 |
"pids": [ |
|
13 |
{ |
|
14 |
"type": "doi", |
|
15 |
"value": "10.000/xyz-123" |
|
16 |
}, |
|
17 |
{ |
|
18 |
"type": "oai", |
|
19 |
"value": "oai:1234" |
|
20 |
} |
|
21 |
], |
|
22 |
"licenseCode": "OPEN", |
|
23 |
"resourceType": "0001", |
|
24 |
"url": "http://test.it/xyz", |
|
25 |
"collectedFromId": "opendoar____::2659", |
|
26 |
"hostedById": "opendoar____::2367", |
|
27 |
"linksToProjects": [ |
|
28 |
"info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus", |
|
29 |
"info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble" |
|
30 |
], |
|
31 |
"contexts": [ |
|
32 |
"egi::classification::natsc::math::pure", |
|
33 |
"egi::classification::natsc::math::stats" |
|
34 |
] |
|
35 |
} |
|
36 |
|
modules/dnet-openaireplus-workflows/trunk/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/submit_test_record.sh | ||
---|---|---|
1 |
#!/bin/bash |
|
2 |
|
|
3 |
curl -H "Content-Type: application/json" --data @test_record.json "http://localhost:8280/app/mvc/api/publications/feedObject" |
modules/dnet-openaireplus-workflows/trunk/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record_complete.json | ||
---|---|---|
1 |
{ |
|
2 |
"originalId": "ORIG_ID_TEST", |
|
3 |
"type":"publication", |
|
4 |
"title": "TEST TITLE", |
|
5 |
"authors": [ |
|
6 |
"Michele Artini", |
|
7 |
"Claudio Atzori", |
|
8 |
"Alessia Bardi" |
|
9 |
], |
|
10 |
"publisher": "Test publisher", |
|
11 |
"description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION", |
|
12 |
"language": "ita", |
|
13 |
"pids": [ |
|
14 |
{ |
|
15 |
"type": "doi", |
|
16 |
"value": "10.000/xyz-123" |
|
17 |
}, |
|
18 |
{ |
|
19 |
"type": "oai", |
|
20 |
"value": "oai:1234" |
|
21 |
} |
|
22 |
], |
|
23 |
"licenseCode": "EMBARGO", |
|
24 |
"embargoEndDate": "2018-02-02", |
|
25 |
"resourceType": "0001", |
|
26 |
"url": "http://test.it/xyz", |
|
27 |
"collectedFromId": "opendoar____::2659", |
|
28 |
"hostedById": "opendoar____::2659", |
|
29 |
"linksToProjects": [ |
|
30 |
"info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus", |
|
31 |
"info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble" |
|
32 |
], |
|
33 |
"contexts": [ |
|
34 |
"egi::classification::natsc::math::pure", |
|
35 |
"egi::classification::natsc::math::stats" |
|
36 |
] |
|
37 |
} |
modules/dnet-openaireplus-workflows/trunk/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record_with_greek_chars.json | ||
---|---|---|
1 |
{ |
|
2 |
"originalId": "ORIG_ID_12345", |
|
3 |
"title": "TEST TITLE WITH Greek Characters", |
|
4 |
"authors": [ |
|
5 |
"αβγδεζηθικλμ νξοπρσςτυφχψω", |
|
6 |
"ΑΒΓΔΕΖΗΘΙΚΛ ΜΝΞΟΠΡΣΤΥΦΧΨΩ" |
|
7 |
], |
|
8 |
"publisher": "âââââââ", |
|
9 |
"description": "Αν περιμένατε να βρίσκεται εδώ μια σελίδα και δεν υπάρχει, η σελίδα μπορεί να μην εμφανίζεται λόγω καθυστέρησης στην ανανέωση της βάσης δεδομένων, ή μπορεί να έχει διαγραφεί. (Δείτε την γρήγορη διαγραφή σελίδων για πιθανούς λόγους). Μπορείτε να δοκιμάστε την λειτουργία εκκαθάρισης, και να ελέγξετε το αρχείο διαγραφών.", |
|
10 |
"language": "ell", |
|
11 |
"pids": [ |
|
12 |
{ |
|
13 |
"type": "doi", |
|
14 |
"value": "10.000/xyz-123-gr" |
|
15 |
} |
|
16 |
], |
|
17 |
"licenseCode": "EMBARGO", |
|
18 |
"embargoEndDate": "2018-02-02", |
|
19 |
"resourceType": "0001", |
|
20 |
"url": "http://test.it/xyz", |
|
21 |
"collectedFromId": "opendoar____::2659", |
|
22 |
"hostedById": "opendoar____::2367", |
|
23 |
"linksToProjects": [ |
|
24 |
"info:eu-repo/grantAgreement/EC/FP7/123456/EU//Test" |
|
25 |
], |
|
26 |
"contexts": [ |
|
27 |
"egi::classification::natsc::math::stats" |
|
28 |
] |
|
29 |
} |
modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/api/OpenaireResultSubmitter.java | ||
---|---|---|
1 |
package eu.dnetlib.msro.openaireplus.api; |
|
2 |
|
|
3 |
import java.io.IOException; |
|
4 |
import java.util.List; |
|
5 |
import javax.annotation.Resource; |
|
6 |
|
|
7 |
import com.google.common.collect.Lists; |
|
8 |
import com.google.gson.Gson; |
|
9 |
import eu.dnetlib.common.rmi.DNetRestDocumentation; |
|
10 |
import eu.dnetlib.data.index.CloudIndexClient; |
|
11 |
import eu.dnetlib.data.index.CloudIndexClientFactory; |
|
12 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; |
|
13 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; |
|
14 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator; |
|
15 |
import eu.dnetlib.msro.openaireplus.api.objects.ResultEntry; |
|
16 |
import eu.dnetlib.msro.openaireplus.utils.OafToIndexRecordFactory; |
|
17 |
import eu.dnetlib.msro.rmi.MSROException; |
|
18 |
import org.apache.commons.io.IOUtils; |
|
19 |
import org.apache.commons.lang.StringUtils; |
|
20 |
import org.apache.commons.lang.exception.ExceptionUtils; |
|
21 |
import org.apache.commons.logging.Log; |
|
22 |
import org.apache.commons.logging.LogFactory; |
|
23 |
import org.apache.velocity.app.VelocityEngine; |
|
24 |
import org.springframework.beans.factory.annotation.Value; |
|
25 |
import org.springframework.core.io.ClassPathResource; |
|
26 |
import org.springframework.http.HttpStatus; |
|
27 |
import org.springframework.stereotype.Controller; |
|
28 |
import org.springframework.web.bind.annotation.*; |
|
29 |
|
|
30 |
/** |
|
31 |
* Created by michele on 11/11/15. |
|
32 |
*/ |
|
33 |
@Controller |
|
34 |
@DNetRestDocumentation |
|
35 |
public class OpenaireResultSubmitter { |
|
36 |
|
|
37 |
private static final Log log = LogFactory.getLog(OpenaireResultSubmitter.class); |
|
38 |
|
|
39 |
public class IndexDsInfo { |
|
40 |
|
|
41 |
private final String indexBaseUrl; |
|
42 |
private final String indexDsId; |
|
43 |
private final String format; |
|
44 |
private final String coll; |
|
45 |
|
|
46 |
public IndexDsInfo(final String indexBaseUrl, final String indexDsId, final String format, final String coll) { |
|
47 |
this.indexBaseUrl = indexBaseUrl; |
|
48 |
this.indexDsId = indexDsId; |
|
49 |
this.format = format; |
|
50 |
this.coll = coll; |
|
51 |
} |
|
52 |
|
|
53 |
public String getIndexBaseUrl() { |
|
54 |
return indexBaseUrl; |
|
55 |
} |
|
56 |
|
|
57 |
public String getIndexDsId() { |
|
58 |
return indexDsId; |
|
59 |
} |
|
60 |
|
|
61 |
public String getFormat() { |
|
62 |
return format; |
|
63 |
} |
|
64 |
|
|
65 |
public String getColl() { |
|
66 |
return coll; |
|
67 |
} |
|
68 |
|
|
69 |
} |
|
70 |
|
|
71 |
@Value(value = "oaf.schema.location") |
|
72 |
private String oafSchemaLocation; |
|
73 |
|
|
74 |
@Resource |
|
75 |
private UniqueServiceLocator serviceLocator; |
|
76 |
|
|
77 |
@Resource |
|
78 |
private OafToIndexRecordFactory oafToIndexRecordFactory; |
|
79 |
|
|
80 |
@Resource |
|
81 |
private RecentResultsQueue recentResultsQueue; |
|
82 |
|
|
83 |
@Resource(name = "openaireplusApisVelocityEngine") |
|
84 |
private VelocityEngine velocityEngine; |
|
85 |
|
|
86 |
@Value(value = "${openaireplus.msro.api.findSolrIndexUrl.xquery}") |
|
87 |
private ClassPathResource findSolrIndexUrl; |
|
88 |
|
|
89 |
@Value(value = "${openaireplus.msro.api.findIndexDsInfo.xquery}") |
|
90 |
private ClassPathResource findIndexDsInfo; |
|
91 |
|
|
92 |
@Deprecated |
|
93 |
@RequestMapping(value = {"/api/publications/feedJson", "/api/results/feedJson"}, method = RequestMethod.POST) |
|
94 |
public |
|
95 |
@ResponseBody |
|
96 |
boolean feedObjectJson(@RequestParam(value = "json", required = true) final String json, |
|
97 |
@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws MSROException { |
|
98 |
final ResultEntry pub = new Gson().fromJson(json, ResultEntry.class); |
|
99 |
return feedObject(pub, commit); |
|
100 |
} |
|
101 |
|
|
102 |
|
|
103 |
@RequestMapping(value = {"/api/results/feedObject"} , method = RequestMethod.POST) |
|
104 |
public |
|
105 |
@ResponseBody |
|
106 |
String feedResult(@RequestBody final ResultEntry pub, |
|
107 |
@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) |
|
108 |
throws MSROException { |
|
109 |
try { |
|
110 |
if(feedObject(pub, commit)) { |
|
111 |
return ResultEntry.calculateObjId(pub.getOriginalId(), pub.getCollectedFromId(), serviceLocator.getService(ISLookUpService.class)); |
|
112 |
} |
|
113 |
else return null; |
|
114 |
} catch (final Throwable e) { |
|
115 |
throw new MSROException("Error adding publication: " + e.getMessage(), e); |
|
116 |
} |
|
117 |
} |
|
118 |
|
|
119 |
@RequestMapping(value = {"/api/publications/feedObject"} , method = RequestMethod.POST) |
|
120 |
public |
|
121 |
@ResponseBody |
|
122 |
boolean feedObject(@RequestBody final ResultEntry pub, |
|
123 |
@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) |
|
124 |
throws MSROException { |
|
125 |
|
|
126 |
checkParameters(pub); |
|
127 |
final List<IndexDsInfo> idxList; |
|
128 |
try { |
|
129 |
idxList = calculateCurrentIndexDsInfo(); |
|
130 |
if (idxList == null || idxList.isEmpty()) { |
|
131 |
throw new MSROException("Cannot add result: " + pub.getOriginalId() + " : No public Search Service found"); |
|
132 |
} |
|
133 |
if (idxList.size() > 1) log.warn("Found more than 1 public search service"); |
|
134 |
final String oafRecord = pub.asOafRecord(velocityEngine, serviceLocator.getService(ISLookUpService.class), oafSchemaLocation); |
|
135 |
|
|
136 |
for (IndexDsInfo idx : idxList) { |
|
137 |
CloudIndexClient idxClient = null; |
|
138 |
try { |
|
139 |
idxClient = CloudIndexClientFactory.newIndexClient(idx.getIndexBaseUrl(), idx.getColl(), false); |
|
140 |
idxClient.feed(oafRecord, idx.getIndexDsId(), oafToIndexRecordFactory.newTransformer(idx.getFormat()), commit); |
|
141 |
} catch (final Throwable e) { |
|
142 |
throw new MSROException("Error adding publication: " + e.getMessage(), e); |
|
143 |
} finally { |
|
144 |
if (idxClient != null) { |
|
145 |
idxClient.close(); |
|
146 |
} |
|
147 |
} |
|
148 |
} |
|
149 |
recentResultsQueue.add(oafRecord); |
|
150 |
return true; |
|
151 |
} catch (final Throwable e) { |
|
152 |
throw new MSROException("Error adding publication: " + e.getMessage(), e); |
|
153 |
} |
|
154 |
|
|
155 |
} |
|
156 |
|
|
157 |
|
|
158 |
|
|
159 |
@RequestMapping(value = "/api/results", method = RequestMethod.DELETE) |
|
160 |
public |
|
161 |
@ResponseBody |
|
162 |
boolean deleteResult( |
|
163 |
@RequestParam(value = "originalId", required = true) final String originalId, |
|
164 |
@RequestParam(value = "collectedFromId", required = true) final String collectedFromId, |
|
165 |
@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws MSROException { |
|
166 |
|
|
167 |
return deleteResultPost(originalId, collectedFromId, commit); |
|
168 |
} |
|
169 |
|
|
170 |
@RequestMapping(value = {"/api/publications/deleteObject", "/api/results/delete"}, method = RequestMethod.POST) |
|
171 |
public |
|
172 |
@ResponseBody |
|
173 |
boolean deleteResultPost( |
|
174 |
@RequestParam(value = "originalId", required = true) final String originalId, |
|
175 |
@RequestParam(value = "collectedFromId", required = true) final String collectedFromId, |
|
176 |
@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws MSROException { |
|
177 |
|
|
178 |
final List<IndexDsInfo> idxList; |
|
179 |
try { |
|
180 |
idxList = calculateCurrentIndexDsInfo(); |
|
181 |
|
|
182 |
if (idxList == null || idxList.isEmpty()) { |
|
183 |
throw new MSROException("Cannot delete result: " + originalId + " : No public Search Service found"); |
|
184 |
} |
|
185 |
if (idxList.size() > 1) log.warn("Found more than 1 public search service"); |
|
186 |
|
|
187 |
final String objId = ResultEntry.calculateObjId(originalId, collectedFromId, serviceLocator.getService(ISLookUpService.class)); |
|
188 |
|
|
189 |
for (IndexDsInfo idx : idxList) { |
|
190 |
CloudIndexClient idxClient = null; |
|
191 |
try { |
|
192 |
idxClient = CloudIndexClientFactory.newIndexClient(idx.getIndexBaseUrl(), idx.getColl(), false); |
|
193 |
idxClient.remove(objId, commit); |
|
194 |
log.info("Deleted result with id: " + objId + " from: " + idx.getIndexBaseUrl()); |
|
195 |
} catch (final Throwable e) { |
|
196 |
throw new MSROException("Error deleting publication: " + e.getMessage(), e); |
|
197 |
} finally { |
|
198 |
if (idxClient != null) { |
|
199 |
idxClient.close(); |
|
200 |
} |
|
201 |
} |
|
202 |
} |
|
203 |
recentResultsQueue.remove(objId); |
|
204 |
return true; |
|
205 |
} catch (IOException | ISLookUpException e) { |
|
206 |
throw new MSROException("Error deleting publication: " + e.getMessage(), e); |
|
207 |
} |
|
208 |
} |
|
209 |
|
|
210 |
@ExceptionHandler(Exception.class) |
|
211 |
@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) |
|
212 |
public |
|
213 |
@ResponseBody |
|
214 |
ErrorMessage handleException(final Exception e) { |
|
215 |
log.error("Error in direct index API", e); |
|
216 |
return new ErrorMessage(e); |
|
217 |
} |
|
218 |
|
|
219 |
private void checkParameters(ResultEntry pub) throws MSROException{ |
|
220 |
if (StringUtils.isBlank(pub.getOriginalId())) { throw new MSROException("A required field is missing: originalId"); } |
|
221 |
if (StringUtils.isBlank(pub.getTitle())) { throw new MSROException("A required field is missing: title"); } |
|
222 |
if (StringUtils.isBlank(pub.getUrl())) { throw new MSROException("A required field is missing: url"); } |
|
223 |
if (StringUtils.isBlank(pub.getLicenseCode())) { throw new MSROException("A required field is missing: licenseCode"); } |
|
224 |
if (StringUtils.isBlank(pub.getResourceType())) { throw new MSROException("A required field is missing: resourceType"); } |
|
225 |
if (StringUtils.isBlank(pub.getCollectedFromId())) { throw new MSROException("A required field is missing: collectedFromId"); } |
|
226 |
if (StringUtils.isBlank(pub.getType())) { throw new MSROException("A required field is missing: type"); } |
|
227 |
} |
|
228 |
|
|
229 |
private List<IndexDsInfo> calculateCurrentIndexDsInfo() throws IOException, ISLookUpException { |
|
230 |
List<IndexDsInfo> list = Lists.newArrayList(); |
|
231 |
|
|
232 |
final String queryUrl = IOUtils.toString(findSolrIndexUrl.getInputStream()); |
|
233 |
final String queryDs = IOUtils.toString(findIndexDsInfo.getInputStream()); |
|
234 |
|
|
235 |
final ISLookUpService lu = serviceLocator.getService(ISLookUpService.class); |
|
236 |
final String indexBaseUrl = lu.getResourceProfileByQuery(queryUrl); |
|
237 |
|
|
238 |
final List<String> idxDs = lu.quickSearchProfile(queryDs); |
|
239 |
for (String idx : idxDs) { |
|
240 |
final String[] arr = idx.split("@@@"); |
|
241 |
list.add(new IndexDsInfo(indexBaseUrl, arr[0].trim(), arr[1].trim(), arr[2].trim())); |
|
242 |
} |
|
243 |
return list; |
|
244 |
} |
|
245 |
|
|
246 |
public class ErrorMessage { |
|
247 |
|
|
248 |
private final String message; |
|
249 |
private final String stacktrace; |
|
250 |
|
|
251 |
public ErrorMessage(final Exception e) { |
|
252 |
this(e.getMessage(), ExceptionUtils.getStackTrace(e)); |
|
253 |
} |
|
254 |
|
|
255 |
public ErrorMessage(final String message, final String stacktrace) { |
|
256 |
this.message = message; |
|
257 |
this.stacktrace = stacktrace; |
|
258 |
} |
|
259 |
|
|
260 |
public String getMessage() { |
|
261 |
return this.message; |
|
262 |
} |
|
263 |
|
|
264 |
public String getStacktrace() { |
|
265 |
return this.stacktrace; |
|
266 |
} |
|
267 |
|
|
268 |
} |
|
269 |
|
|
270 |
} |
modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/api/RecentPublicationsQueue.java | ||
---|---|---|
1 |
package eu.dnetlib.msro.openaireplus.api; |
|
2 |
|
|
3 |
import java.io.StringReader; |
|
4 |
import java.util.Iterator; |
|
5 |
import java.util.List; |
|
6 |
|
|
7 |
import org.apache.commons.logging.Log; |
|
8 |
import org.apache.commons.logging.LogFactory; |
|
9 |
import org.dom4j.io.SAXReader; |
|
10 |
import org.springframework.beans.factory.annotation.Required; |
|
11 |
|
|
12 |
import com.mongodb.BasicDBObject; |
|
13 |
import com.mongodb.BasicDBObjectBuilder; |
|
14 |
import com.mongodb.DB; |
|
15 |
import com.mongodb.DBCollection; |
|
16 |
import com.mongodb.DBCursor; |
|
17 |
import com.mongodb.DBObject; |
|
18 |
|
|
19 |
import eu.dnetlib.miscutils.datetime.DateUtils; |
|
20 |
|
|
21 |
/** |
|
22 |
* Created by michele on 11/11/15. |
|
23 |
*/ |
|
24 |
public class RecentPublicationsQueue implements Iterable<String> { |
|
25 |
|
|
26 |
private static final Log log = LogFactory.getLog(RecentPublicationsQueue.class); |
|
27 |
|
|
28 |
private DB db; |
|
29 |
private String collection; |
|
30 |
|
|
31 |
public void init() { |
|
32 |
if (!db.collectionExists(collection)) { |
|
33 |
log.info(String.format("creating collection %s", collection)); |
|
34 |
db.createCollection(collection, new BasicDBObject()); |
|
35 |
} |
|
36 |
} |
|
37 |
|
|
38 |
@Override |
|
39 |
public Iterator<String> iterator() { |
|
40 |
|
|
41 |
final DBCursor cursor = db.getCollection(collection).find(); |
|
42 |
|
|
43 |
return new Iterator<String>() { |
|
44 |
|
|
45 |
@Override |
|
46 |
public boolean hasNext() { |
|
47 |
return cursor.hasNext(); |
|
48 |
} |
|
49 |
|
|
50 |
@Override |
|
51 |
public String next() { |
|
52 |
final DBObject obj = cursor.next(); |
|
53 |
return ((obj != null) && obj.containsField("record")) ? obj.get("record").toString() : ""; |
|
54 |
} |
|
55 |
|
|
56 |
@Override |
|
57 |
public void remove() { |
|
58 |
throw new RuntimeException("NOT IMPLEMENTED"); |
|
59 |
} |
|
60 |
}; |
|
61 |
} |
|
62 |
|
|
63 |
synchronized public void add(final String oaf) throws Exception { |
|
64 |
final String id = (new SAXReader()).read(new StringReader(oaf)).valueOf("//*[local-name() = 'objIdentifier']"); |
|
65 |
|
|
66 |
log.info("Saving record " + id + " in db: " + db.getName() + ", coll: " + collection); |
|
67 |
|
|
68 |
final DBCollection coll = db.getCollection(collection); |
|
69 |
final DBObject obj = BasicDBObjectBuilder.start() |
|
70 |
.append("id", id) |
|
71 |
.append("record", oaf) |
|
72 |
.append("date", DateUtils.now()) |
|
73 |
.get(); |
|
74 |
coll.update(new BasicDBObject("id", id), obj, true, false); |
|
75 |
} |
|
76 |
|
|
77 |
public void remove(final List<String> list) { |
|
78 |
final DBCollection coll = db.getCollection(collection); |
|
79 |
for (final String id : list) { |
|
80 |
coll.remove(new BasicDBObject("id", id)); |
|
81 |
} |
|
82 |
} |
|
83 |
|
|
84 |
public void remove(final String... ids) { |
|
85 |
final DBCollection coll = db.getCollection(collection); |
|
86 |
for (final String id : ids) { |
|
87 |
coll.remove(new BasicDBObject("id", id)); |
|
88 |
} |
|
89 |
} |
|
90 |
|
|
91 |
public DB getDb() { |
|
92 |
return db; |
|
93 |
} |
|
94 |
|
|
95 |
@Required |
|
96 |
public void setDb(final DB db) { |
|
97 |
this.db = db; |
|
98 |
} |
|
99 |
|
|
100 |
public String getCollection() { |
|
101 |
return collection; |
|
102 |
} |
|
103 |
|
|
104 |
@Required |
|
105 |
public void setCollection(final String collection) { |
|
106 |
this.collection = collection; |
|
107 |
} |
|
108 |
} |
modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/api/OpenAIRESubmitterUtils.java | ||
---|---|---|
1 |
package eu.dnetlib.msro.openaireplus.api; |
|
2 |
|
|
3 |
import java.io.StringWriter; |
|
4 |
import java.util.ArrayList; |
|
5 |
import java.util.HashMap; |
|
6 |
import java.util.List; |
|
7 |
import java.util.Map; |
|
8 |
|
|
9 |
import org.apache.commons.lang.StringUtils; |
|
10 |
|
|
11 |
import com.google.common.base.Function; |
|
12 |
import com.google.common.collect.Lists; |
|
13 |
|
|
14 |
import eu.dnetlib.miscutils.functional.hash.Hashing; |
|
15 |
|
|
16 |
/** |
|
17 |
* Created by michele on 15/01/16. |
|
18 |
*/ |
|
19 |
public class OpenAIRESubmitterUtils { |
|
20 |
|
|
21 |
public Map<String, String> calculateProjectInfo(final String link) { |
|
22 |
final Map<String, String> info = new HashMap<String, String>(); |
|
23 |
final String[] arr = link.split("/"); |
|
24 |
// info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble |
|
25 |
|
|
26 |
if (arr.length > 4) { |
|
27 |
final String acronym = arr.length > 7 ? arr[7] : ""; |
|
28 |
final String title = arr.length > 6 ? StringUtils.isNotBlank(arr[6]) ? arr[6] : acronym : ""; |
|
29 |
final String jurisdiction = arr.length > 5 ? arr[5] : ""; |
|
30 |
final String funderId = calculateFunderId(arr[2]); |
|
31 |
info.put("id", calculateProjectId(arr[2], arr[3], arr[4])); |
|
32 |
info.put("funderShortName", arr[2]); |
|
33 |
info.put("fundingName", arr[3]); |
|
34 |
info.put("code", arr[4]); |
|
35 |
info.put("jurisdiction", jurisdiction); |
|
36 |
info.put("title", title); |
|
37 |
info.put("acronym", acronym); |
|
38 |
info.put("funderId", funderId); |
|
39 |
info.put("funderName", calculateFunderName(arr[2])); |
|
40 |
info.put("fundingId", funderId + "::" + arr[3]); |
|
41 |
} |
|
42 |
return info; |
|
43 |
} |
|
44 |
|
|
45 |
private String calculateProjectId(final String funderShortName, final String funding, final String code) { |
|
46 |
final String suffix = Hashing.md5(code); |
|
47 |
|
|
48 |
if (funderShortName.equalsIgnoreCase("ec")) { |
|
49 |
if (funding.equalsIgnoreCase("fp7")) { |
|
50 |
return "corda_______::" + suffix; |
|
51 |
} else { |
|
52 |
return "corda__h2020::" + suffix; |
|
53 |
} |
|
54 |
} else if (funderShortName.equalsIgnoreCase("arc")) { |
|
55 |
return "arc_________::" + suffix; |
|
56 |
} else if (funderShortName.equalsIgnoreCase("fct")) { |
|
57 |
return "fct_________::" + suffix; |
|
58 |
} else if (funderShortName.equalsIgnoreCase("nhmrc")) { |
|
59 |
return "nhmrc_______::" + suffix; |
|
60 |
} else if (funderShortName.equalsIgnoreCase("wt")) { |
|
61 |
return "wt__________::" + suffix; |
|
62 |
} else { |
|
63 |
return ""; |
|
64 |
} |
|
65 |
} |
|
66 |
|
|
67 |
private String calculateFunderId(final String funderShortName) { |
|
68 |
if (funderShortName.equalsIgnoreCase("ec")) { |
|
69 |
return "ec__________::EC"; |
|
70 |
} else if (funderShortName.equalsIgnoreCase("arc")) { |
|
71 |
return "arc_________::ARC"; |
|
72 |
} else if (funderShortName.equalsIgnoreCase("fct")) { |
|
73 |
return "fct_________::FCT"; |
|
74 |
} else if (funderShortName.equalsIgnoreCase("nhmrc")) { |
|
75 |
return "nhmrc_______::NHMRC"; |
|
76 |
} else if (funderShortName.equalsIgnoreCase("wt")) { |
|
77 |
return "wt__________::WT"; |
|
78 |
} else { |
|
79 |
return ""; |
|
80 |
} |
|
81 |
} |
|
82 |
|
|
83 |
private String calculateFunderName(final String funderShortName) { |
|
84 |
if (funderShortName.equalsIgnoreCase("ec")) { |
|
85 |
return "European Commission"; |
|
86 |
} else if (funderShortName.equalsIgnoreCase("arc")) { |
|
87 |
return "Australian Research Council (ARC)"; |
|
88 |
} else if (funderShortName.equalsIgnoreCase("fct")) { |
|
89 |
return "Fundação para a Ciência e a Tecnologia"; |
|
90 |
} else if (funderShortName.equalsIgnoreCase("nhmrc")) { |
|
91 |
return "National Health and Medical Research Council (NHMRC)"; |
|
92 |
} else if (funderShortName.equalsIgnoreCase("wt")) { |
|
93 |
return "Wellcome Trust"; |
|
94 |
} else { |
|
95 |
return ""; |
|
96 |
} |
|
97 |
} |
|
98 |
|
|
99 |
public List<ContextInfo> processContexts(final List<String> list) { |
|
100 |
return Lists.newArrayList(Lists.transform(list, new Function<String, ContextInfo>() { |
|
101 |
|
|
102 |
@Override |
|
103 |
public ContextInfo apply(final String s) { |
|
104 |
return createContextInfo(s.split("::"), 0); |
|
105 |
} |
|
106 |
|
|
107 |
private ContextInfo createContextInfo(final String[] arr, final int pos) { |
|
108 |
final StringWriter id = new StringWriter(); |
|
109 |
id.write(arr[0]); |
|
110 |
for (int i = 0; i < pos; i++) { |
|
111 |
id.write("::"); |
|
112 |
id.write(arr[i + 1]); |
|
113 |
} |
|
114 |
final String elem = (pos == 0) ? "context" : (pos == 1) ? "category" : "concept"; |
|
115 |
final ContextInfo info = new ContextInfo(elem, id.toString()); |
|
116 |
if ((pos + 1) < arr.length) { |
|
117 |
info.getChildren().add(createContextInfo(arr, pos + 1)); |
|
118 |
} |
|
119 |
return info; |
|
120 |
} |
|
121 |
})); |
|
122 |
|
|
123 |
} |
|
124 |
|
|
125 |
public class ContextInfo { |
|
126 |
|
|
127 |
private String elem; |
|
128 |
private String id; |
|
129 |
private List<ContextInfo> children = new ArrayList<ContextInfo>(); |
|
130 |
|
|
131 |
public ContextInfo(final String elem, |
|
132 |
final String id) { |
|
133 |
this.elem = elem; |
|
134 |
this.id = id; |
|
135 |
} |
|
136 |
|
|
137 |
public String getElem() { |
|
138 |
return elem; |
|
139 |
} |
|
140 |
|
|
141 |
public void setElem(final String elem) { |
|
142 |
this.elem = elem; |
|
143 |
} |
|
144 |
|
|
145 |
public String getId() { |
|
146 |
return id; |
|
147 |
} |
|
148 |
|
|
149 |
public void setId(final String id) { |
|
150 |
this.id = id; |
|
151 |
} |
|
152 |
|
|
153 |
public List<ContextInfo> getChildren() { |
|
154 |
return children; |
|
155 |
} |
|
156 |
|
|
157 |
public void setChildren(final List<ContextInfo> children) { |
|
158 |
this.children = children; |
|
159 |
} |
|
160 |
|
|
161 |
public boolean isRoot() { |
|
162 |
return elem.equals("context"); |
|
163 |
} |
|
164 |
} |
|
165 |
} |
modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/api/RecentResultsQueue.java | ||
---|---|---|
1 |
package eu.dnetlib.msro.openaireplus.api; |
|
2 |
|
|
3 |
import java.io.StringReader; |
|
4 |
import java.util.Iterator; |
|
5 |
import java.util.List; |
|
6 |
|
|
7 |
import org.apache.commons.logging.Log; |
|
8 |
import org.apache.commons.logging.LogFactory; |
|
9 |
import org.dom4j.io.SAXReader; |
|
10 |
import org.springframework.beans.factory.annotation.Required; |
|
11 |
|
|
12 |
import com.mongodb.BasicDBObject; |
|
13 |
import com.mongodb.BasicDBObjectBuilder; |
|
14 |
import com.mongodb.DB; |
|
15 |
import com.mongodb.DBCollection; |
|
16 |
import com.mongodb.DBCursor; |
|
17 |
import com.mongodb.DBObject; |
|
18 |
|
|
19 |
import eu.dnetlib.miscutils.datetime.DateUtils; |
|
20 |
|
|
21 |
/** |
|
22 |
* Created by michele on 11/11/15. |
|
23 |
*/ |
|
24 |
public class RecentResultsQueue implements Iterable<String> { |
|
25 |
|
|
26 |
private static final Log log = LogFactory.getLog(RecentResultsQueue.class); |
|
27 |
|
|
28 |
private DB db; |
|
29 |
private String collection; |
|
30 |
|
|
31 |
public void init() { |
|
32 |
if (!db.collectionExists(collection)) { |
|
33 |
log.info(String.format("creating collection %s", collection)); |
|
34 |
db.createCollection(collection, new BasicDBObject()); |
|
35 |
} |
|
36 |
} |
|
37 |
|
|
38 |
@Override |
|
39 |
public Iterator<String> iterator() { |
|
40 |
|
|
41 |
final DBCursor cursor = db.getCollection(collection).find(); |
|
42 |
|
|
43 |
return new Iterator<String>() { |
|
44 |
|
|
45 |
@Override |
|
46 |
public boolean hasNext() { |
|
47 |
return cursor.hasNext(); |
|
48 |
} |
|
49 |
|
|
50 |
@Override |
|
51 |
public String next() { |
|
52 |
final DBObject obj = cursor.next(); |
|
53 |
return ((obj != null) && obj.containsField("record")) ? obj.get("record").toString() : ""; |
|
54 |
} |
|
55 |
|
|
56 |
@Override |
|
57 |
public void remove() { |
|
58 |
throw new RuntimeException("NOT IMPLEMENTED"); |
|
59 |
} |
|
60 |
}; |
|
61 |
} |
|
62 |
|
|
63 |
synchronized public void add(final String oaf) throws Exception { |
|
64 |
final String id = (new SAXReader()).read(new StringReader(oaf)).valueOf("//*[local-name() = 'objIdentifier']"); |
|
65 |
|
|
66 |
log.info("Saving record " + id + " in db: " + db.getName() + ", coll: " + collection); |
|
67 |
|
|
68 |
final DBCollection coll = db.getCollection(collection); |
|
69 |
final DBObject obj = BasicDBObjectBuilder.start() |
|
70 |
.append("id", id) |
|
71 |
.append("record", oaf) |
|
72 |
.append("date", DateUtils.now()) |
|
73 |
.get(); |
|
74 |
coll.update(new BasicDBObject("id", id), obj, true, false); |
|
75 |
} |
|
76 |
|
|
77 |
public void remove(final List<String> list) { |
|
78 |
final DBCollection coll = db.getCollection(collection); |
|
79 |
for (final String id : list) { |
|
80 |
coll.remove(new BasicDBObject("id", id)); |
|
81 |
} |
|
82 |
} |
|
83 |
|
|
84 |
public void remove(final String... ids) { |
|
85 |
final DBCollection coll = db.getCollection(collection); |
|
86 |
for (final String id : ids) { |
|
87 |
coll.remove(new BasicDBObject("id", id)); |
|
88 |
} |
|
89 |
} |
|
90 |
|
|
91 |
public DB getDb() { |
|
92 |
return db; |
|
93 |
} |
|
94 |
|
|
95 |
@Required |
|
96 |
public void setDb(final DB db) { |
|
97 |
this.db = db; |
|
98 |
} |
|
99 |
|
|
100 |
public String getCollection() { |
|
101 |
return collection; |
|
102 |
} |
|
103 |
|
|
104 |
@Required |
|
105 |
public void setCollection(final String collection) { |
|
106 |
this.collection = collection; |
|
107 |
} |
|
108 |
} |
modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/api/objects/ResultEntry.java | ||
---|---|---|
1 |
package eu.dnetlib.msro.openaireplus.api.objects; |
|
2 |
|
|
3 |
import java.text.SimpleDateFormat; |
|
4 |
import java.util.ArrayList; |
|
5 |
import java.util.Date; |
|
6 |
import java.util.HashMap; |
|
7 |
import java.util.List; |
|
8 |
import java.util.Map; |
|
9 |
import java.util.concurrent.TimeUnit; |
|
10 |
|
|
11 |
//import io.swagger.annotations.ApiModelProperty; |
|
12 |
import org.apache.commons.lang.StringUtils; |
|
13 |
import org.apache.commons.logging.Log; |
|
14 |
import org.apache.commons.logging.LogFactory; |
|
15 |
import org.apache.velocity.app.VelocityEngine; |
|
16 |
import org.apache.velocity.tools.generic.EscapeTool; |
|
17 |
import org.springframework.ui.velocity.VelocityEngineUtils; |
|
18 |
|
|
19 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; |
|
20 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; |
|
21 |
import eu.dnetlib.miscutils.datetime.DateUtils; |
|
22 |
import eu.dnetlib.miscutils.functional.hash.Hashing; |
|
23 |
import eu.dnetlib.msro.openaireplus.api.OpenAIRESubmitterUtils; |
|
24 |
|
|
25 |
/** |
|
26 |
* Created by michele on 02/12/15. |
|
27 |
*/ |
|
28 |
public class ResultEntry { |
|
29 |
|
|
30 |
private String originalId; |
|
31 |
private String title; |
|
32 |
private List<String> authors = new ArrayList<String>(); |
|
33 |
private String publisher; |
|
34 |
private String description; |
|
35 |
private String language; |
|
36 |
private List<PidEntry> pids = new ArrayList<PidEntry>(); |
|
37 |
private String licenseCode; |
|
38 |
private String embargoEndDate; |
|
39 |
private String type = "publication"; |
|
40 |
private String resourceType; |
|
41 |
private String url; |
|
42 |
private String collectedFromId; |
|
43 |
private String hostedById; |
|
44 |
|
|
45 |
// String according to openaire guidelines: |
|
46 |
// info:eu-repo/grantAgreement/Funder/FundingProgram/ProjectID/[Jurisdiction]/[ProjectName]/[ProjectAcronym] |
|
47 |
private List<String> contexts = new ArrayList<String>(); |
|
48 |
|
|
49 |
// String according to the EGI context profile, example: egi::classification::natsc::math |
|
50 |
private List<String> linksToProjects = new ArrayList<String>(); |
|
51 |
|
|
52 |
private static long last_cache_update = 0; |
|
53 |
private static final Map<String, Map<String, String>> cached_vocabularies = new HashMap<String, Map<String, String>>(); |
|
54 |
private static final Map<String, DatasourceEntry> cached_datasources = new HashMap<String, DatasourceEntry>(); |
|
55 |
private static final Map<String, String> cached_contexts = new HashMap<String, String>(); |
|
56 |
|
|
57 |
private static final Log log = LogFactory.getLog(ResultEntry.class); |
|
58 |
|
|
59 |
public ResultEntry() {} |
|
60 |
|
|
61 |
// @ApiModelProperty(required = true) |
|
62 |
public String getOriginalId() { |
|
63 |
return originalId; |
|
64 |
} |
|
65 |
|
|
66 |
public void setOriginalId(final String originalId) { |
|
67 |
this.originalId = originalId; |
|
68 |
} |
|
69 |
|
|
70 |
// @ApiModelProperty(required = true) |
|
71 |
public String getTitle() { |
|
72 |
return title; |
|
73 |
} |
|
74 |
|
|
75 |
public void setTitle(final String title) { |
|
76 |
this.title = title; |
|
77 |
} |
|
78 |
|
|
79 |
|
|
80 |
public List<String> getAuthors() { |
|
81 |
return authors; |
|
82 |
} |
|
83 |
|
|
84 |
public void setAuthors(final List<String> authors) { |
|
85 |
this.authors = authors; |
|
86 |
} |
|
87 |
|
|
88 |
public String getPublisher() { |
|
89 |
return publisher; |
|
90 |
} |
|
91 |
|
|
92 |
public void setPublisher(final String publisher) { |
|
93 |
this.publisher = publisher; |
|
94 |
} |
|
95 |
|
|
96 |
public String getDescription() { |
|
97 |
return description; |
|
98 |
} |
|
99 |
|
|
100 |
public void setDescription(final String description) { |
|
101 |
this.description = description; |
|
102 |
} |
|
103 |
|
|
104 |
// @ApiModelProperty(value="ISO Alpha-3 code. E.g. 'eng', 'ita'") |
|
105 |
public String getLanguage() { |
|
106 |
return language; |
|
107 |
} |
|
108 |
|
|
109 |
public void setLanguage(final String language) { |
|
110 |
this.language = language; |
|
111 |
} |
|
112 |
|
|
113 |
public List<PidEntry> getPids() { |
|
114 |
return pids; |
|
115 |
} |
|
116 |
|
|
117 |
public void setPids(final List<PidEntry> pids) { |
|
118 |
this.pids = pids; |
|
119 |
} |
|
120 |
|
|
121 |
// @ApiModelProperty(required = true, allowableValues = "OPEN, CLOSED, RESTRICTED, EMBARGO, UNKNOWN, OTHER") |
|
122 |
public String getLicenseCode() { |
|
123 |
return licenseCode; |
|
124 |
} |
|
125 |
|
|
126 |
public void setLicenseCode(final String licenseCode) { |
|
127 |
this.licenseCode = licenseCode; |
|
128 |
} |
|
129 |
|
|
130 |
// @ApiModelProperty(required = true, value="Use 001 for articles, 021 for datasets. See: http://api.openaire.eu/vocabularies/dnet:publication_resource.") |
|
131 |
public String getResourceType() { |
|
132 |
return resourceType; |
|
133 |
} |
|
134 |
|
|
135 |
public void setResourceType(final String resourceType) { |
|
136 |
this.resourceType = resourceType; |
|
137 |
} |
|
138 |
|
|
139 |
// @ApiModelProperty(required = true) |
|
140 |
public String getUrl() { |
|
141 |
return url; |
|
142 |
} |
|
143 |
|
|
144 |
public void setUrl(final String url) { |
|
145 |
this.url = url; |
|
146 |
} |
|
147 |
|
|
148 |
// @ApiModelProperty(required = true, value="Use opendoar___::2659 for Zenodo Publications; re3data_____::r3d100010468 for Zenodo datasets; infrastruct::openaire for OpenAIRE portal.") |
|
149 |
public String getCollectedFromId() { |
|
150 |
return collectedFromId; |
|
151 |
} |
|
152 |
|
|
153 |
public void setCollectedFromId(final String collectedFromId) { |
|
154 |
this.collectedFromId = collectedFromId; |
|
155 |
} |
|
156 |
|
|
157 |
public String getHostedById() { |
|
158 |
return hostedById; |
|
159 |
} |
|
160 |
|
|
161 |
public void setHostedById(final String hostedById) { |
|
162 |
this.hostedById = hostedById; |
|
163 |
} |
|
164 |
|
|
165 |
// @ApiModelProperty(value="E.g. fet, egi::classification::natsc::math::pure, egi::projects::EMI") |
|
166 |
public List<String> getContexts() { |
|
167 |
return contexts; |
|
168 |
} |
|
169 |
|
|
170 |
public void setContexts(final List<String> contexts) { |
|
171 |
this.contexts = contexts; |
|
172 |
} |
|
173 |
// @ApiModelProperty(value="E.g. info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus") |
|
174 |
public List<String> getLinksToProjects() { |
|
175 |
return linksToProjects; |
|
176 |
} |
|
177 |
|
|
178 |
public void setLinksToProjects(final List<String> linksToProjects) { |
|
179 |
this.linksToProjects = linksToProjects; |
|
180 |
} |
|
181 |
|
|
182 |
// @ApiModelProperty(allowableValues = "publication, dataset") |
|
183 |
public String getType() { |
|
184 |
return type; |
|
185 |
} |
|
186 |
|
|
187 |
public void setType(final String type) { |
|
188 |
this.type = type; |
|
189 |
} |
|
190 |
|
|
191 |
public String getEmbargoEndDate() { |
|
192 |
return embargoEndDate; |
|
193 |
} |
|
194 |
|
|
195 |
public void setEmbargoEndDate(final String embargoEndDate) { |
|
196 |
this.embargoEndDate = embargoEndDate; |
|
197 |
} |
|
198 |
|
|
199 |
public String asOafRecord(final VelocityEngine ve, |
|
200 |
final ISLookUpService lookupService, |
|
201 |
final String oafSchemaLocation) throws Exception { |
|
202 |
|
|
203 |
final DatasourceEntry collectedFromEntry = getDatasourceInfo(collectedFromId, lookupService); |
|
204 |
final DatasourceEntry hostedByEntry = getDatasourceInfo(hostedById, lookupService); |
|
205 |
|
|
206 |
final String objId = calculateObjId(originalId, collectedFromEntry); |
|
207 |
|
|
208 |
final Map<String, Object> model = new HashMap<String, Object>(); |
|
209 |
model.put("esc", new EscapeTool()); |
|
210 |
model.put("util", new OpenAIRESubmitterUtils()); |
|
211 |
model.put("pub", this); |
|
212 |
model.put("objIdentifier", objId); |
|
213 |
model.put("oafSchemaLocation", oafSchemaLocation); |
|
214 |
model.put("resultTypes", getVocabulary("dnet:result_typologies", lookupService)); |
|
215 |
model.put("licenses", getVocabulary("dnet:access_modes", lookupService)); |
|
216 |
model.put("resourceTypes", getVocabulary("dnet:publication_resource", lookupService)); |
|
217 |
model.put("pidTypes", getVocabulary("dnet:pid_types", lookupService)); |
|
218 |
model.put("languages", getVocabulary("dnet:languages", lookupService)); |
|
219 |
model.put("contexts", getContexts(lookupService)); |
|
220 |
model.put("dateOfCollection", (new SimpleDateFormat("yyyy-MM-dd\'T\'hh:mm:ss\'Z\'")).format(new Date())); |
|
221 |
model.put("collectedFrom", collectedFromEntry); |
|
222 |
model.put("hostedBy", hostedByEntry); |
|
223 |
|
|
224 |
return VelocityEngineUtils.mergeTemplateIntoString(ve, "/eu/dnetlib/msro/openaireplus/api/indexRecord.xml.vm", "UTF-8", model); |
|
225 |
} |
|
226 |
|
|
227 |
public static String calculateObjId(final String originalId, final DatasourceEntry collectedFromEntry) { |
|
228 |
return collectedFromEntry.getPrefix() + "::" + Hashing.md5(originalId); |
|
229 |
} |
|
230 |
|
|
231 |
public static String calculateObjId(final String originalId, final String collectedFromId, final ISLookUpService lookupService) throws ISLookUpException { |
|
232 |
return calculateObjId(originalId, getDatasourceInfo(collectedFromId, lookupService)); |
|
233 |
} |
|
234 |
|
|
235 |
private synchronized static DatasourceEntry getDatasourceInfo(final String dsId, final ISLookUpService lookupService) throws ISLookUpException { |
|
236 |
if (StringUtils |
|
237 |
.isBlank(dsId)) { return new DatasourceEntry("openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18", "Unknown Repository", "unknown_____"); } |
|
238 |
|
|
239 |
if (!cached_datasources.containsKey(dsId)) { |
|
240 |
final String query = |
|
241 |
"collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')//CONFIGURATION[./DATASOURCE_ORIGINAL_ID='" + dsId |
|
242 |
+ "']/concat(./OFFICIAL_NAME, ' @@@ ', .//FIELD/value[../key='NamespacePrefix'])"; |
|
243 |
final String s = lookupService.getResourceProfileByQuery(query); |
|
244 |
final String[] arr = s.split("@@@"); |
|
245 |
|
|
246 |
final DatasourceEntry ds = new DatasourceEntry(dsId, arr[0].trim(), arr[1].trim()); |
|
247 |
|
|
248 |
if (StringUtils.isBlank(ds.getName()) || StringUtils.isBlank(ds.getPrefix())) { |
|
249 |
log.error("Invalid datasource id: " + dsId); |
|
250 |
throw new ISLookUpException("Invalid datasource id: " + dsId); |
|
251 |
} else { |
|
252 |
cached_datasources.put(dsId, ds); |
|
253 |
} |
|
254 |
} |
|
255 |
|
|
256 |
return cached_datasources.get(dsId); |
|
257 |
|
|
258 |
} |
|
259 |
|
|
260 |
private synchronized static Map<String, String> getVocabulary(final String voc, final ISLookUpService lookupService) throws ISLookUpException { |
|
261 |
|
|
262 |
if (((DateUtils.now() - last_cache_update) < TimeUnit.MINUTES.toMillis(15)) && cached_vocabularies.containsKey(voc)) { |
|
263 |
return cached_vocabularies.get(voc); |
|
264 |
} else { |
|
265 |
final String query = "collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType')[.//VOCABULARY_NAME/@code='" + voc |
|
266 |
+ "']//TERM/concat(@code, ' @@@ ', @english_name)"; |
|
267 |
|
|
268 |
final Map<String, String> map = new HashMap<String, String>(); |
|
269 |
for (final String s : lookupService.quickSearchProfile(query)) { |
|
270 |
final String[] arr = s.split("@@@"); |
|
271 |
map.put(arr[0].trim(), arr[1].trim()); |
|
272 |
} |
|
273 |
|
|
274 |
cached_vocabularies.put(voc, map); |
|
275 |
|
|
276 |
last_cache_update = DateUtils.now(); |
|
277 |
|
|
278 |
return map; |
|
279 |
} |
|
280 |
} |
|
281 |
|
|
282 |
private synchronized static Map<String, String> getContexts(final ISLookUpService lookupService) throws ISLookUpException { |
|
283 |
if (((DateUtils.now() - last_cache_update) > TimeUnit.MINUTES.toMillis(15)) || cached_contexts.isEmpty()) { |
|
284 |
final String query = |
|
285 |
"collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')[.//context/@type='community']//*[name()='context' or name()='category' or name()='concept']/concat(@id, ' @@@ ', @label)"; |
|
286 |
|
|
287 |
cached_contexts.clear(); |
|
288 |
for (final String s : lookupService.quickSearchProfile(query)) { |
|
289 |
final String[] arr = s.split("@@@"); |
|
290 |
cached_contexts.put(arr[0].trim(), arr[1].trim()); |
|
291 |
} |
|
292 |
last_cache_update = DateUtils.now(); |
|
293 |
} |
|
294 |
return cached_contexts; |
|
295 |
} |
|
296 |
|
|
297 |
} |
modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/api/objects/DatasourceEntry.java | ||
---|---|---|
1 |
package eu.dnetlib.msro.openaireplus.api.objects; |
|
2 |
|
|
3 |
import eu.dnetlib.miscutils.functional.hash.Hashing; |
|
4 |
import org.apache.commons.lang.StringUtils; |
|
5 |
|
|
6 |
/** |
|
7 |
* Created by michele on 02/12/15. |
|
8 |
*/ |
|
9 |
public class DatasourceEntry { |
|
10 |
|
|
11 |
private String id; |
|
12 |
private String name; |
|
13 |
private String prefix; |
|
14 |
|
|
15 |
public DatasourceEntry() { |
|
16 |
} |
|
17 |
|
|
18 |
public DatasourceEntry(final String id, final String name, final String prefix) { |
|
19 |
this.id = id; |
|
20 |
this.name = name; |
|
21 |
this.prefix = prefix; |
|
22 |
} |
|
23 |
|
|
24 |
public String getId() { |
|
25 |
return id; |
|
26 |
} |
|
27 |
|
|
28 |
public void setId(final String id) { |
|
29 |
this.id = id; |
|
30 |
} |
|
31 |
|
|
32 |
public String getName() { |
|
33 |
return name; |
|
34 |
} |
|
35 |
|
|
36 |
public void setName(final String name) { |
|
37 |
this.name = name; |
|
38 |
} |
|
39 |
|
|
40 |
public String getPrefix() { |
|
41 |
return prefix; |
|
42 |
} |
|
43 |
|
|
44 |
public void setPrefix(final String prefix) { |
|
45 |
this.prefix = prefix; |
|
46 |
} |
|
47 |
|
|
48 |
public String calculateOpenaireId() { |
|
49 |
if (StringUtils.isNotBlank(id)) { |
|
50 |
final String[] arr = id.split("::"); |
|
51 |
if (arr.length == 2) { |
|
52 |
return String.format("%s::%s", arr[0], Hashing.md5(arr[1])); |
|
53 |
} |
|
54 |
} |
|
55 |
return ""; |
|
56 |
} |
|
57 |
} |
modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/api/objects/PidEntry.java | ||
---|---|---|
1 |
package eu.dnetlib.msro.openaireplus.api.objects; |
|
2 |
|
|
3 |
//import io.swagger.annotations.ApiModelProperty; |
|
4 |
|
|
5 |
/** |
|
6 |
* Created by michele on 02/12/15. |
|
7 |
*/ |
|
8 |
public class PidEntry { |
|
9 |
|
|
10 |
private String type; |
|
11 |
private String value; |
|
12 |
|
|
13 |
public PidEntry() { |
|
14 |
} |
|
15 |
|
|
16 |
public PidEntry(final String type, final String value) { |
|
17 |
this.type = type; |
|
18 |
this.value = value; |
|
19 |
} |
|
20 |
//@ApiModelProperty(required = true, value="E.g. doi, pmc, urn. See http://api.openaire.eu/vocabularies/dnet:pid_types") |
|
21 |
public String getType() { |
|
22 |
return type; |
|
23 |
} |
|
24 |
|
|
25 |
public void setType(final String type) { |
|
26 |
this.type = type; |
|
27 |
} |
|
28 |
|
|
29 |
//@ApiModelProperty(required = true) |
|
30 |
public String getValue() { |
|
31 |
return value; |
|
32 |
} |
|
33 |
|
|
34 |
public void setValue(final String value) { |
|
35 |
this.value = value; |
|
36 |
} |
|
37 |
} |
modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/msro/openaireplus/api/indexRecord.xml.vm | ||
---|---|---|
1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
2 |
|
|
3 |
<!-- |
|
4 |
#macro(context $ctx) |
|
5 |
<$!ctx.elem id="$!esc.xml($!ctx.id)" label="$!esc.xml($contexts.get($!ctx.id))" #if($!ctx.isRoot()) type="community"#end> |
|
6 |
#foreach($child in $!ctx.children)#context($child)#end |
|
7 |
</$!ctx.elem> |
|
8 |
#end |
|
9 |
--> |
|
10 |
|
|
11 |
<record xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:oaf="http://namespace.openaire.eu/oaf"> |
|
12 |
<result> |
|
13 |
<header> |
|
14 |
<dri:objIdentifier>$!esc.xml($!objIdentifier)</dri:objIdentifier> |
|
15 |
<dri:dateOfCollection>$!esc.xml($!dateOfCollection)</dri:dateOfCollection> |
|
16 |
<dri:status>under curation</dri:status> |
|
17 |
<counters/> |
|
18 |
</header> |
|
19 |
<metadata> |
|
20 |
<oaf:entity schemaLocation="http://namespace.openaire.eu/oaf $!oafSchemaLocation"> |
|
21 |
<oaf:result> |
|
22 |
<title classid="main title" classname="main title" schemeid="dnet:dataCite_title" schemename="dnet:dataCite_title"> |
|
23 |
$!esc.xml($!pub.title) |
|
24 |
</title> |
|
25 |
<dateofacceptance/> |
|
26 |
<resulttype classid="$!esc.xml($!pub.type)" classname="$!esc.xml($!resultTypes.get($!pub.type))" schemeid="dnet:result_typologies" schemename="dnet:result_typologies"/> |
|
27 |
<language classid="$!esc.xml($!pub.language)" classname="$!esc.xml($!languages.get($!pub.language))" schemeid="dnet:languages" |
|
28 |
schemename="dnet:languages"/> |
|
29 |
<description> |
|
30 |
$!esc.xml($!pub.description) |
|
31 |
</description> |
|
32 |
<country classid="" classname="" schemeid="" schemename=""/> |
|
33 |
<subject classid="" classname="" schemeid="" schemename=""/> |
|
34 |
<relevantdate classid="" classname="" schemeid="" schemename=""/> |
|
35 |
<publisher>$!esc.xml($!pub.publisher)</publisher> |
|
36 |
<embargoenddate>$!esc.xml($!pub.embargoEndDate)</embargoenddate> |
|
37 |
<source/> |
|
38 |
<fulltext/> |
|
39 |
<format/> |
|
40 |
<storagedate/> |
|
41 |
<resourcetype classid="" classname="" schemeid="" schemename=""/> |
|
42 |
<device/> |
|
43 |
<size/> |
|
44 |
<version/> |
|
45 |
<lastmetadataupdate/> |
|
46 |
<metadataversionnumber/> |
|
47 |
<originalId>$!esc.xml($!pub.originalId)</originalId> |
|
48 |
<collectedfrom name="$!esc.xml($!collectedFrom.name)" id="$!esc.xml($!collectedFrom.calculateOpenaireId())"/> |
|
49 |
|
|
50 |
#foreach($pid in $!pub.pids) |
|
51 |
<pid classid="$!esc.xml($!pid.type)" classname="$!esc.xml($!pidTypes.get($!pid.type))" schemeid="dnet:pid_types" |
|
52 |
schemename="dnet:pid_types">$!esc.xml($pid.value)</pid> |
|
53 |
#end |
|
54 |
|
|
55 |
<bestlicense classid="$!esc.xml($!pub.licenseCode)" classname="$!esc.xml($!licenses.get($!pub.licenseCode))" |
|
56 |
schemeid="dnet:access_modes" schemename="dnet:access_modes"/> |
|
57 |
|
|
58 |
#foreach($ctx in $util.processContexts($!pub.contexts)) |
|
59 |
#context($ctx) |
|
60 |
#end |
|
61 |
|
|
62 |
<datainfo> |
|
63 |
<inferred>false</inferred> |
|
64 |
<deletedbyinference>false</deletedbyinference> |
|
65 |
<trust>0.9</trust> |
|
66 |
<inferenceprovenance/> |
|
67 |
<provenanceaction classid="user:insert" classname="user:insert" schemeid="dnet:provenanceActions" schemename="dnet:provenanceActions"/> |
|
68 |
</datainfo> |
|
69 |
|
|
70 |
<rels> |
|
71 |
#foreach($author in $!pub.authors) |
|
72 |
<rel inferred="false" trust="0.9" inferenceprovenance="" provenanceaction="sysimport:crosswalk:repository"> |
|
73 |
<to class="hasAuthor" scheme="dnet:personroles" type="person"/> |
|
74 |
<ranking>$velocityCount</ranking> |
|
75 |
<fullname>$!author</fullname> |
|
76 |
</rel> |
|
77 |
#end |
Also available in: Unified diff
updated the direct index api