Revision 50937
Added by Alessia Bardi about 6 years ago
CatalogueRegistrator.java | ||
---|---|---|
10 | 10 |
import com.fasterxml.jackson.core.JsonEncoding; |
11 | 11 |
import com.fasterxml.jackson.core.JsonFactory; |
12 | 12 |
import com.fasterxml.jackson.core.JsonGenerator; |
13 |
import com.google.common.collect.Lists; |
|
13 | 14 |
import eu.dnetlib.parthenos.CRM; |
14 | 15 |
import eu.dnetlib.parthenos.CRMdig; |
15 | 16 |
import eu.dnetlib.parthenos.CRMpe; |
16 | 17 |
import eu.dnetlib.parthenos.publisher.ParthenosPublisherException; |
17 | 18 |
import eu.dnetlib.parthenos.rdf.ResourceReader; |
18 |
import org.apache.commons.io.IOUtils; |
|
19 | 19 |
import org.apache.commons.lang3.StringUtils; |
20 | 20 |
import org.apache.commons.logging.Log; |
21 | 21 |
import org.apache.commons.logging.LogFactory; |
22 |
import org.apache.http.client.methods.HttpGet; |
|
23 |
import org.apache.http.client.methods.HttpRequestBase; |
|
24 | 22 |
import org.apache.http.client.utils.URIBuilder; |
25 | 23 |
import org.apache.jena.ontology.OntModel; |
26 | 24 |
import org.apache.jena.ontology.OntModelSpec; |
27 |
import org.apache.jena.rdf.model.InfModel; |
|
28 | 25 |
import org.apache.jena.rdf.model.ModelFactory; |
29 |
import org.apache.jena.rdf.model.ResIterator; |
|
30 | 26 |
import org.apache.jena.rdf.model.Resource; |
31 |
import org.apache.jena.vocabulary.RDF; |
|
32 | 27 |
import org.springframework.beans.factory.annotation.Autowired; |
33 | 28 |
import org.springframework.beans.factory.annotation.Value; |
29 |
import org.springframework.http.*; |
|
34 | 30 |
import org.springframework.stereotype.Component; |
31 |
import org.springframework.web.client.RestTemplate; |
|
35 | 32 |
|
33 |
import static eu.dnetlib.parthenos.CRM.*; |
|
34 |
|
|
36 | 35 |
/** |
37 | 36 |
* Created by Alessia Bardi on 21/11/2017. |
38 | 37 |
* |
... | ... | |
42 | 41 |
public class CatalogueRegistrator { |
43 | 42 |
|
44 | 43 |
private static final Log log = LogFactory.getLog(CatalogueRegistrator.class); |
45 |
private OntModel baseModel; |
|
46 | 44 |
|
47 | 45 |
@Value("${gcube.catalogue.baseurl}") |
48 | 46 |
private String baseURL; |
... | ... | |
52 | 50 |
@Autowired |
53 | 51 |
private ResourceReader resourceReader; |
54 | 52 |
|
55 |
@PostConstruct |
|
56 |
public void init(){ |
|
57 |
baseModel = ModelFactory.createOntologyModel(OntModelSpec.RDFS_MEM_TRANS_INF); |
|
58 |
baseModel.read(CRMpe.RDFS_URL); |
|
59 |
baseModel.read(CRM.RDFS_URL); |
|
60 |
baseModel.read(CRMdig.RDFS_URL); |
|
61 |
} |
|
53 |
@Autowired |
|
54 |
private RestTemplate jrrRestTemplate; |
|
62 | 55 |
|
63 |
public int register(final String rdfRecord, final String objIdentifier) throws ParthenosPublisherException, IOException { |
|
64 |
log.info("Catalogue --> Processing record " + objIdentifier); |
|
65 |
int registered = 0; |
|
66 |
int total = 0; |
|
67 |
InfModel model = loadBaseModel(); |
|
68 |
model.read(IOUtils.toInputStream(rdfRecord, "UTF-8"), CRMpe.NS); |
|
69 | 56 |
|
70 |
|
|
71 |
ResIterator activities = model.listSubjectsWithProperty(RDF.type, CRM.E7_Activity); |
|
72 |
log.debug("Registering instances of E7_Activity"); |
|
73 |
while (activities.hasNext()) { |
|
74 |
total++; |
|
75 |
Resource activity = activities.nextResource(); |
|
76 |
String resURI = activity.getURI(); |
|
77 |
if(!resURI.startsWith("http")){ |
|
78 |
//this is something George said: if it has no http URI, then it is not to be considered relevant by itself |
|
79 |
log.info("E7_Activity "+resURI+" skipped: URI does not start with http"); |
|
80 |
} |
|
81 |
String resCatName = resURI.substring(resURI.lastIndexOf("/") + 1); |
|
82 |
String json = getJsonForActivity(activity, resCatName ); |
|
83 |
registered += doRegister(json, objIdentifier, resCatName); |
|
57 |
public String register(final Resource resource, final Resource type) throws IOException, ParthenosPublisherException { |
|
58 |
String resURI = resource.getURI(); |
|
59 |
log.debug(String.format("Catalogue --> Processing resource : %s with type: %s", resURI, type.getLocalName())); |
|
60 |
String resCatName = resURI.substring(resURI.lastIndexOf("/") + 1); |
|
61 |
String json; |
|
62 |
switch(type.getLocalName()){ |
|
63 |
case "E7_Activity": |
|
64 |
json = getJsonForActivity(resource, resCatName); |
|
65 |
break; |
|
66 |
case "E39_Actor": |
|
67 |
json = getJsonForActor(resource, resCatName); |
|
68 |
break; |
|
69 |
case "E70_Thing": |
|
70 |
json = getJsonForThing(resource, resCatName); |
|
71 |
break; |
|
72 |
//TODO: handle more tyopes, see JRRPublisher |
|
73 |
default: |
|
74 |
throw new IllegalArgumentException(String.format("Type "+type.getLocalName()+" not supported")); |
|
84 | 75 |
} |
85 |
|
|
86 |
ResIterator actors = model.listSubjectsWithProperty(RDF.type, CRM.E39_Actor); |
|
87 |
log.debug("Registering instances of E39_Actor"); |
|
88 |
while (actors.hasNext()) { |
|
89 |
total++; |
|
90 |
Resource actor = actors.nextResource(); |
|
91 |
String resURI = actor.getURI(); |
|
92 |
if(!resURI.startsWith("http")){ |
|
93 |
//this is something George said: if it has no http URI, then it is not to be considered relevant by itself |
|
94 |
log.info("E39_Actor "+resURI+" skipped: URI does not start with http"); |
|
95 |
} |
|
96 |
String resCatName = resURI.substring(resURI.lastIndexOf("/") + 1); |
|
97 |
String json = getJsonForActor(actor, resCatName ); |
|
98 |
registered += doRegister(json, objIdentifier, resCatName); |
|
99 |
} |
|
100 |
//TODO: nothing to do for E29 and Curation plan subclass? |
|
101 |
//ResIterator procs = model.listResourcesWithProperty(RDF.type, CRM.E29_Design_or_Procedure); |
|
102 |
|
|
103 |
ResIterator things = model.listSubjectsWithProperty(RDF.type, CRM.E70_Thing); |
|
104 |
log.debug("Registering instances of E70_Thing"); |
|
105 |
while (things.hasNext()) { |
|
106 |
total++; |
|
107 |
Resource thing = things.nextResource(); |
|
108 |
String resURI = thing.getURI(); |
|
109 |
if(!resURI.startsWith("http")){ |
|
110 |
//this is something George said: if it has no http URI, then it is not to be considered relevant by itself |
|
111 |
log.info("E70_Thing "+resURI+" skipped: URI does not start with http"); |
|
112 |
} |
|
113 |
String resCatName = resURI.substring(resURI.lastIndexOf("/") + 1); |
|
114 |
String json = getJsonForThing(thing, resCatName ); |
|
115 |
registered += doRegister(json, objIdentifier, resCatName); |
|
116 |
} |
|
117 |
log.debug(String.format("Registering finished: %s/%s", registered, total)); |
|
118 |
return registered; |
|
76 |
String uuid = doRegister(json, resCatName); |
|
77 |
log.debug(String.format("%s registered on the catalogue with uuid: %s", resURI, uuid)); |
|
78 |
return uuid; |
|
119 | 79 |
} |
120 | 80 |
|
121 |
protected String getJsonForThing(final Resource res, final String resNameForCatalogue) throws IOException { |
|
122 |
JsonFactory jsonFactory = new JsonFactory(); |
|
123 |
final ByteArrayOutputStream out = new ByteArrayOutputStream(); |
|
124 |
BufferedOutputStream bos = new BufferedOutputStream(out); |
|
125 |
JsonGenerator jg = jsonFactory.createGenerator(bos, JsonEncoding.UTF8); |
|
126 |
jg.writeStartObject(); |
|
127 |
writeCommonFields(jg, res, resNameForCatalogue); |
|
128 | 81 |
|
129 |
jg.writeArrayFieldStart("extras"); |
|
130 |
addExtra(jg, "system:type", CRM.E70_Thing.getLocalName()); |
|
131 |
//specific class |
|
132 |
addExtra(jg,"instance of", resourceReader.findSpecificType(res, CRM.E70_Thing).getLocalName()); |
|
133 |
if(res.getURI().startsWith("http://parthenos.d4science.org")) { |
|
134 |
addExtra(jg,"Parthenos URL", res.getURI()); |
|
135 |
} |
|
136 |
//TODO: things include digital objects, software, datasets, schema. Guess we should know what to add here. |
|
137 |
jg.writeEndArray(); |
|
138 |
|
|
139 |
jg.writeEndObject(); |
|
140 |
jg.close(); |
|
141 |
return out.toString("UTF-8"); |
|
142 |
} |
|
143 |
|
|
144 |
protected void addExtra(final JsonGenerator jg, final String key, final String value) throws IOException { |
|
145 |
jg.writeStartObject(); |
|
146 |
jg.writeStringField("key", key ); |
|
147 |
jg.writeStringField("value", value); |
|
148 |
jg.writeEndObject(); |
|
149 |
} |
|
150 |
|
|
151 |
protected void writeCommonFields(final JsonGenerator jg, final Resource res, final String resNameForCatalogue) throws IOException { |
|
152 |
//end of URI |
|
153 |
jg.writeStringField("name", resNameForCatalogue); |
|
154 |
//default license |
|
155 |
jg.writeStringField("license_id", "notspecified"); |
|
156 |
String title = resourceReader.getTitle(res); |
|
157 |
if(StringUtils.isBlank(title)) |
|
158 |
title = resNameForCatalogue; |
|
159 |
jg.writeStringField("title", title); |
|
160 |
//description |
|
161 |
jg.writeStringField("notes", resourceReader.getDescription(res)); |
|
162 |
} |
|
163 |
|
|
164 |
protected String getJsonForActor(final Resource res, final String resNameForCatalogue) throws IOException { |
|
165 |
JsonFactory jsonFactory = new JsonFactory(); |
|
166 |
final ByteArrayOutputStream out = new ByteArrayOutputStream(); |
|
167 |
BufferedOutputStream bos = new BufferedOutputStream(out); |
|
168 |
JsonGenerator jg = jsonFactory.createGenerator(bos, JsonEncoding.UTF8); |
|
169 |
jg.writeStartObject(); |
|
170 |
writeCommonFields(jg, res, resNameForCatalogue); |
|
171 |
|
|
172 |
jg.writeArrayFieldStart("extras"); |
|
173 |
addExtra(jg,"system:type", CRM.E39_Actor.getLocalName()); |
|
174 |
//specific class |
|
175 |
addExtra(jg,"instance of", resourceReader.findSpecificType(res, CRM.E39_Actor).getLocalName()); |
|
176 |
if(res.getURI().startsWith("http://parthenos.d4science.org")) |
|
177 |
addExtra(jg,"Parthenos URL", res.getURI()); |
|
178 |
jg.writeEndArray(); |
|
179 |
|
|
180 |
jg.writeEndObject(); |
|
181 |
jg.close(); |
|
182 |
return out.toString("UTF-8"); |
|
183 |
} |
|
184 |
|
|
185 | 82 |
protected String getJsonForActivity(final Resource res, final String resNameForCatalogue) throws IOException { |
186 | 83 |
JsonFactory jsonFactory = new JsonFactory(); |
187 | 84 |
final ByteArrayOutputStream out = new ByteArrayOutputStream(); |
... | ... | |
195 | 92 |
//the names of all superclasses of the entity |
196 | 93 |
jg.writeArrayFieldStart("tags"); |
197 | 94 |
Iterator<String> classNames = resourceReader.getRDFClassNames(res); |
198 |
while(classNames.hasNext()) { |
|
95 |
while (classNames.hasNext()) {
|
|
199 | 96 |
jg.writeStartObject(); |
200 | 97 |
jg.writeStringField("name", classNames.next()); |
201 | 98 |
jg.writeEndObject(); |
... | ... | |
213 | 110 |
jg.writeEndArray(); |
214 | 111 |
|
215 | 112 |
jg.writeArrayFieldStart("extras"); |
216 |
addExtra(jg,"system:type", CRM.E7_Activity.getLocalName());
|
|
113 |
addExtra(jg, "system:type", E7_Activity.getLocalName());
|
|
217 | 114 |
//specific class |
218 |
addExtra(jg,"instance of", resourceReader.findSpecificType(res, CRM.E7_Activity).getLocalName());
|
|
219 |
if(res.getURI().startsWith("http://parthenos.d4science.org")) |
|
220 |
addExtra(jg,"Parthenos URL", res.getURI()); |
|
221 |
addExtra(jg,"competence", res.getURI()); |
|
115 |
addExtra(jg, "instance of", resourceReader.findSpecificType(res, E7_Activity).getLocalName());
|
|
116 |
if (res.getURI().startsWith("http://parthenos.d4science.org"))
|
|
117 |
addExtra(jg, "Parthenos URL", res.getURI());
|
|
118 |
addExtra(jg, "competence", res.getURI());
|
|
222 | 119 |
//TODO: from which linked E39_Actors? Should be the same as for groups above |
223 |
addExtra(jg,"provided by", ""); |
|
120 |
addExtra(jg, "provided by", "");
|
|
224 | 121 |
//TODO: get begin/end of operation from PP42_has_declarative_time |
225 |
addExtra(jg,"declared begin/end of operation", ""); |
|
122 |
addExtra(jg, "declared begin/end of operation", "");
|
|
226 | 123 |
//TODO: where to get it? |
227 |
addExtra(jg,"last confirmation", ""); |
|
124 |
addExtra(jg, "last confirmation", "");
|
|
228 | 125 |
//TODO: where to get it? |
229 |
addExtra(jg,"date of registration", ""); |
|
230 |
addExtra(jg,"availability", resourceReader.getAvailability(res)); |
|
231 |
addExtra(jg,"condition of use", resourceReader.getConditionOfUse(res)); |
|
126 |
addExtra(jg, "date of registration", "");
|
|
127 |
addExtra(jg, "availability", resourceReader.getAvailability(res));
|
|
128 |
addExtra(jg, "condition of use", resourceReader.getConditionOfUse(res));
|
|
232 | 129 |
//TODO: where to get the contact person |
233 | 130 |
//Resource contactPerson = null; |
234 | 131 |
//jg.writeStringField("contact person", resourceReader.getTitle(contactPerson)); |
235 | 132 |
//TODO: where to get it? |
236 |
addExtra(jg,"activity type", ""); |
|
133 |
addExtra(jg, "activity type", "");
|
|
237 | 134 |
// can't add multiple fields with same name, hence skipping |
238 | 135 |
//jg.writeStringField("hosts", ""); |
239 | 136 |
//TODO: where to get it? |
240 |
addExtra(jg,"online access point", ""); |
|
137 |
addExtra(jg, "online access point", "");
|
|
241 | 138 |
//TODO: where to get it? |
242 |
addExtra(jg,"authorization", ""); |
|
139 |
addExtra(jg, "authorization", "");
|
|
243 | 140 |
//TODO: where to get it? |
244 |
addExtra(jg,"protocol", ""); |
|
141 |
addExtra(jg, "protocol", "");
|
|
245 | 142 |
//can't add multiple fields with same name, hence skipping |
246 | 143 |
//addExtra(jg,"curates", ""); |
247 | 144 |
//TODO: where to get it? |
248 |
addExtra(jg,"runs on request", ""); |
|
145 |
addExtra(jg, "runs on request", "");
|
|
249 | 146 |
//TODO: where to get it? |
250 |
addExtra(jg,"delivers on request", ""); |
|
147 |
addExtra(jg, "delivers on request", "");
|
|
251 | 148 |
jg.writeEndArray(); //end extras |
252 | 149 |
|
253 | 150 |
jg.writeEndObject(); |
254 | 151 |
jg.close(); |
255 | 152 |
return out.toString("UTF-8"); |
256 | 153 |
} |
154 |
protected String getJsonForActor(final Resource res, final String resNameForCatalogue) throws IOException { |
|
155 |
JsonFactory jsonFactory = new JsonFactory(); |
|
156 |
final ByteArrayOutputStream out = new ByteArrayOutputStream(); |
|
157 |
BufferedOutputStream bos = new BufferedOutputStream(out); |
|
158 |
JsonGenerator jg = jsonFactory.createGenerator(bos, JsonEncoding.UTF8); |
|
159 |
jg.writeStartObject(); |
|
160 |
writeCommonFields(jg, res, resNameForCatalogue); |
|
257 | 161 |
|
162 |
jg.writeArrayFieldStart("extras"); |
|
163 |
addExtra(jg, "system:type", E39_Actor.getLocalName()); |
|
164 |
//specific class |
|
165 |
addExtra(jg, "instance of", resourceReader.findSpecificType(res, E39_Actor).getLocalName()); |
|
166 |
if (res.getURI().startsWith("http://parthenos.d4science.org")) |
|
167 |
addExtra(jg, "Parthenos URL", res.getURI()); |
|
168 |
jg.writeEndArray(); |
|
258 | 169 |
|
259 |
private int doRegister(final String json, final String objIdentifier, final String catName) throws ParthenosPublisherException { |
|
260 |
//TODO: implement me |
|
261 |
log.debug(String.format("Catalogue --> Registering %s (from %s) : %s", catName, objIdentifier, json)); |
|
262 |
//if resCatName is already registered --> delete it and then saves it again |
|
263 |
try { |
|
264 |
URI uri = new URIBuilder(getBaseURL()).build(); |
|
265 |
HttpGet req = new HttpGet(uri); |
|
266 |
setHeaders(req); |
|
267 |
System.out.println(req.getURI()); |
|
268 |
return 0; |
|
269 |
}catch(Throwable t){ |
|
270 |
throw new ParthenosPublisherException(t); |
|
170 |
jg.writeEndObject(); |
|
171 |
jg.close(); |
|
172 |
return out.toString("UTF-8"); |
|
173 |
} |
|
174 |
protected String getJsonForThing(final Resource res, final String resNameForCatalogue) throws IOException { |
|
175 |
JsonFactory jsonFactory = new JsonFactory(); |
|
176 |
final ByteArrayOutputStream out = new ByteArrayOutputStream(); |
|
177 |
BufferedOutputStream bos = new BufferedOutputStream(out); |
|
178 |
JsonGenerator jg = jsonFactory.createGenerator(bos, JsonEncoding.UTF8); |
|
179 |
jg.writeStartObject(); |
|
180 |
writeCommonFields(jg, res, resNameForCatalogue); |
|
181 |
|
|
182 |
jg.writeArrayFieldStart("extras"); |
|
183 |
addExtra(jg, "system:type", E70_Thing.getLocalName()); |
|
184 |
//specific class |
|
185 |
addExtra(jg, "instance of", resourceReader.findSpecificType(res, E70_Thing).getLocalName()); |
|
186 |
if (res.getURI().startsWith("http://parthenos.d4science.org")) { |
|
187 |
addExtra(jg, "Parthenos URL", res.getURI()); |
|
271 | 188 |
} |
189 |
//TODO: things include digital objects, software, datasets, schema. Guess we should know what to add here. |
|
190 |
jg.writeEndArray(); |
|
191 |
|
|
192 |
jg.writeEndObject(); |
|
193 |
jg.close(); |
|
194 |
return out.toString("UTF-8"); |
|
272 | 195 |
} |
273 | 196 |
|
274 |
private void setHeaders(final HttpRequestBase req){ |
|
275 |
req.setHeader("Content-type", "application/json"); |
|
276 |
req.setHeader("Accept","application/json"); |
|
277 |
req.setHeader("gcube-token", getApplicationToken() ); |
|
197 |
protected void addExtra(final JsonGenerator jg, final String key, final String value) throws IOException { |
|
198 |
jg.writeStartObject(); |
|
199 |
jg.writeStringField("key", key); |
|
200 |
jg.writeStringField("value", value); |
|
201 |
jg.writeEndObject(); |
|
278 | 202 |
} |
279 | 203 |
|
280 |
protected InfModel loadBaseModel() { |
|
281 |
return ModelFactory.createRDFSModel(baseModel); |
|
204 |
protected void writeCommonFields(final JsonGenerator jg, final Resource res, final String resNameForCatalogue) throws IOException { |
|
205 |
//end of URI |
|
206 |
jg.writeStringField("name", resNameForCatalogue); |
|
207 |
//default license |
|
208 |
jg.writeStringField("license_id", "notspecified"); |
|
209 |
String title = resourceReader.getTitle(res); |
|
210 |
if (StringUtils.isBlank(title)) |
|
211 |
title = resNameForCatalogue; |
|
212 |
jg.writeStringField("title", title); |
|
213 |
//description |
|
214 |
jg.writeStringField("notes", resourceReader.getDescription(res)); |
|
282 | 215 |
} |
283 | 216 |
|
284 |
public OntModel getBaseModel() { |
|
285 |
return baseModel; |
|
217 |
/** |
|
218 |
* @param json |
|
219 |
* @param resCatName |
|
220 |
* @return the uuid assigned by the catalogue to the registered resource |
|
221 |
* @throws ParthenosPublisherException |
|
222 |
*/ |
|
223 |
private String doRegister(final String json, final String resCatName) throws ParthenosPublisherException { |
|
224 |
//TODO: implement me |
|
225 |
log.debug(String.format("Catalogue --> Registering %s : %s", resCatName, json)); |
|
226 |
//if resCatName is already registered --> delete it and then saves it again |
|
227 |
HttpEntity<String> entity = new HttpEntity<String>(json, getHeaders()); |
|
228 |
try { |
|
229 |
URI uri = new URIBuilder(getBaseURL()).addParameter("gcube-token", getApplicationToken()).build(); |
|
230 |
ResponseEntity<String> res = jrrRestTemplate.<String>exchange(uri, HttpMethod.POST, entity, String.class); |
|
231 |
//TODO: I guess response is a json, need to check the format, parse it to get the uuid of the registered resource |
|
232 |
String uuid = res.getBody(); |
|
233 |
return uuid; |
|
234 |
} catch (Throwable t) { |
|
235 |
throw new ParthenosPublisherException(t); |
|
236 |
} |
|
286 | 237 |
} |
287 | 238 |
|
288 |
public void setBaseModel(final OntModel baseModel) { |
|
289 |
this.baseModel = baseModel; |
|
239 |
private HttpHeaders getHeaders() { |
|
240 |
HttpHeaders headers = new HttpHeaders(); |
|
241 |
headers.setContentType(MediaType.APPLICATION_JSON); |
|
242 |
headers.setAccept(Lists.newArrayList(MediaType.APPLICATION_JSON)); |
|
243 |
//headers.set("gcube-token", getApplicationToken() ); |
|
244 |
return headers; |
|
290 | 245 |
} |
291 | 246 |
|
247 |
|
|
292 | 248 |
public String getBaseURL() { |
293 | 249 |
return baseURL; |
294 | 250 |
} |
Also available in: Unified diff
Refactoring and deprecation for registration on catalogue and registry