Project

General

Profile

« Previous | Next » 

Revision 50937

Refactoring and deprecation for registration on catalogue and registry

View differences:

CatalogueRegistrator.java
10 10
import com.fasterxml.jackson.core.JsonEncoding;
11 11
import com.fasterxml.jackson.core.JsonFactory;
12 12
import com.fasterxml.jackson.core.JsonGenerator;
13
import com.google.common.collect.Lists;
13 14
import eu.dnetlib.parthenos.CRM;
14 15
import eu.dnetlib.parthenos.CRMdig;
15 16
import eu.dnetlib.parthenos.CRMpe;
16 17
import eu.dnetlib.parthenos.publisher.ParthenosPublisherException;
17 18
import eu.dnetlib.parthenos.rdf.ResourceReader;
18
import org.apache.commons.io.IOUtils;
19 19
import org.apache.commons.lang3.StringUtils;
20 20
import org.apache.commons.logging.Log;
21 21
import org.apache.commons.logging.LogFactory;
22
import org.apache.http.client.methods.HttpGet;
23
import org.apache.http.client.methods.HttpRequestBase;
24 22
import org.apache.http.client.utils.URIBuilder;
25 23
import org.apache.jena.ontology.OntModel;
26 24
import org.apache.jena.ontology.OntModelSpec;
27
import org.apache.jena.rdf.model.InfModel;
28 25
import org.apache.jena.rdf.model.ModelFactory;
29
import org.apache.jena.rdf.model.ResIterator;
30 26
import org.apache.jena.rdf.model.Resource;
31
import org.apache.jena.vocabulary.RDF;
32 27
import org.springframework.beans.factory.annotation.Autowired;
33 28
import org.springframework.beans.factory.annotation.Value;
29
import org.springframework.http.*;
34 30
import org.springframework.stereotype.Component;
31
import org.springframework.web.client.RestTemplate;
35 32

  
33
import static eu.dnetlib.parthenos.CRM.*;
34

  
36 35
/**
37 36
 * Created by Alessia Bardi on 21/11/2017.
38 37
 *
......
42 41
public class CatalogueRegistrator {
43 42

  
44 43
	private static final Log log = LogFactory.getLog(CatalogueRegistrator.class);
45
	private OntModel baseModel;
46 44

  
47 45
	@Value("${gcube.catalogue.baseurl}")
48 46
	private String baseURL;
......
52 50
	@Autowired
53 51
	private ResourceReader resourceReader;
54 52

  
55
	@PostConstruct
56
	public void init(){
57
		baseModel = ModelFactory.createOntologyModel(OntModelSpec.RDFS_MEM_TRANS_INF);
58
		baseModel.read(CRMpe.RDFS_URL);
59
		baseModel.read(CRM.RDFS_URL);
60
		baseModel.read(CRMdig.RDFS_URL);
61
	}
53
	@Autowired
54
	private RestTemplate jrrRestTemplate;
62 55

  
63
	public int register(final String rdfRecord, final String objIdentifier) throws ParthenosPublisherException, IOException {
64
		log.info("Catalogue --> Processing record " + objIdentifier);
65
		int registered = 0;
66
		int total = 0;
67
		InfModel model = loadBaseModel();
68
		model.read(IOUtils.toInputStream(rdfRecord, "UTF-8"), CRMpe.NS);
69 56

  
70

  
71
		ResIterator activities = model.listSubjectsWithProperty(RDF.type, CRM.E7_Activity);
72
		log.debug("Registering instances of E7_Activity");
73
		while (activities.hasNext()) {
74
			total++;
75
			Resource activity = activities.nextResource();
76
			String resURI = activity.getURI();
77
			if(!resURI.startsWith("http")){
78
				//this is something George said: if it has no http URI, then it is not to be considered relevant by itself
79
				log.info("E7_Activity "+resURI+" skipped: URI does not start with http");
80
			}
81
			String resCatName = resURI.substring(resURI.lastIndexOf("/") + 1);
82
			String json = getJsonForActivity(activity, resCatName );
83
			registered += doRegister(json, objIdentifier, resCatName);
57
	public String register(final Resource resource, final Resource type) throws IOException, ParthenosPublisherException {
58
		String resURI = resource.getURI();
59
		log.debug(String.format("Catalogue --> Processing resource : %s with type: %s", resURI, type.getLocalName()));
60
		String resCatName = resURI.substring(resURI.lastIndexOf("/") + 1);
61
		String json;
62
		switch(type.getLocalName()){
63
		case "E7_Activity":
64
			json = getJsonForActivity(resource, resCatName);
65
			break;
66
		case "E39_Actor":
67
			json = getJsonForActor(resource, resCatName);
68
			break;
69
		case "E70_Thing":
70
			json = getJsonForThing(resource, resCatName);
71
			break;
72
			//TODO: handle more tyopes, see JRRPublisher
73
		default:
74
			throw new IllegalArgumentException(String.format("Type "+type.getLocalName()+" not supported"));
84 75
		}
85

  
86
		ResIterator actors = model.listSubjectsWithProperty(RDF.type, CRM.E39_Actor);
87
		log.debug("Registering instances of E39_Actor");
88
		while (actors.hasNext()) {
89
			total++;
90
			Resource actor = actors.nextResource();
91
			String resURI = actor.getURI();
92
			if(!resURI.startsWith("http")){
93
				//this is something George said: if it has no http URI, then it is not to be considered relevant by itself
94
				log.info("E39_Actor "+resURI+" skipped: URI does not start with http");
95
			}
96
			String resCatName = resURI.substring(resURI.lastIndexOf("/") + 1);
97
			String json = getJsonForActor(actor, resCatName );
98
			registered += doRegister(json, objIdentifier, resCatName);
99
		}
100
		//TODO: nothing to do for E29 and Curation plan subclass?
101
		//ResIterator procs = model.listResourcesWithProperty(RDF.type, CRM.E29_Design_or_Procedure);
102

  
103
		ResIterator things = model.listSubjectsWithProperty(RDF.type, CRM.E70_Thing);
104
		log.debug("Registering instances of E70_Thing");
105
		while (things.hasNext()) {
106
			total++;
107
			Resource thing = things.nextResource();
108
			String resURI = thing.getURI();
109
			if(!resURI.startsWith("http")){
110
				//this is something George said: if it has no http URI, then it is not to be considered relevant by itself
111
				log.info("E70_Thing "+resURI+" skipped: URI does not start with http");
112
			}
113
			String resCatName = resURI.substring(resURI.lastIndexOf("/") + 1);
114
			String json = getJsonForThing(thing, resCatName );
115
			registered += doRegister(json, objIdentifier, resCatName);
116
		}
117
		log.debug(String.format("Registering finished: %s/%s", registered, total));
118
		return registered;
76
		String uuid = doRegister(json, resCatName);
77
		log.debug(String.format("%s registered on the catalogue with uuid: %s", resURI, uuid));
78
		return uuid;
119 79
	}
120 80

  
121
	protected String getJsonForThing(final Resource res, final String resNameForCatalogue) throws IOException {
122
		JsonFactory jsonFactory = new JsonFactory();
123
		final ByteArrayOutputStream out = new ByteArrayOutputStream();
124
		BufferedOutputStream bos = new BufferedOutputStream(out);
125
		JsonGenerator jg = jsonFactory.createGenerator(bos, JsonEncoding.UTF8);
126
		jg.writeStartObject();
127
		writeCommonFields(jg, res, resNameForCatalogue);
128 81

  
129
		jg.writeArrayFieldStart("extras");
130
		addExtra(jg, "system:type", CRM.E70_Thing.getLocalName());
131
		//specific class
132
		addExtra(jg,"instance of", resourceReader.findSpecificType(res, CRM.E70_Thing).getLocalName());
133
		if(res.getURI().startsWith("http://parthenos.d4science.org")) {
134
			addExtra(jg,"Parthenos URL", res.getURI());
135
		}
136
		//TODO: things include digital objects, software, datasets, schema. Guess we should know what to add here.
137
		jg.writeEndArray();
138

  
139
		jg.writeEndObject();
140
		jg.close();
141
		return out.toString("UTF-8");
142
	}
143

  
144
	protected void addExtra(final JsonGenerator jg, final String key, final String value) throws IOException {
145
		jg.writeStartObject();
146
		jg.writeStringField("key", key );
147
		jg.writeStringField("value", value);
148
		jg.writeEndObject();
149
	}
150

  
151
	protected void writeCommonFields(final JsonGenerator jg, final Resource res, final String resNameForCatalogue) throws IOException {
152
		//end of URI
153
		jg.writeStringField("name", resNameForCatalogue);
154
		//default license
155
		jg.writeStringField("license_id", "notspecified");
156
		String title = resourceReader.getTitle(res);
157
		if(StringUtils.isBlank(title))
158
			title = resNameForCatalogue;
159
		jg.writeStringField("title", title);
160
		//description
161
		jg.writeStringField("notes", resourceReader.getDescription(res));
162
	}
163

  
164
	protected String getJsonForActor(final Resource res, final String resNameForCatalogue) throws IOException {
165
		JsonFactory jsonFactory = new JsonFactory();
166
		final ByteArrayOutputStream out = new ByteArrayOutputStream();
167
		BufferedOutputStream bos = new BufferedOutputStream(out);
168
		JsonGenerator jg = jsonFactory.createGenerator(bos, JsonEncoding.UTF8);
169
		jg.writeStartObject();
170
		writeCommonFields(jg, res, resNameForCatalogue);
171

  
172
		jg.writeArrayFieldStart("extras");
173
		addExtra(jg,"system:type", CRM.E39_Actor.getLocalName());
174
		//specific class
175
		addExtra(jg,"instance of", resourceReader.findSpecificType(res, CRM.E39_Actor).getLocalName());
176
		if(res.getURI().startsWith("http://parthenos.d4science.org"))
177
			addExtra(jg,"Parthenos URL", res.getURI());
178
		jg.writeEndArray();
179

  
180
		jg.writeEndObject();
181
		jg.close();
182
		return out.toString("UTF-8");
183
	}
184

  
185 82
	protected String getJsonForActivity(final Resource res, final String resNameForCatalogue) throws IOException {
186 83
		JsonFactory jsonFactory = new JsonFactory();
187 84
		final ByteArrayOutputStream out = new ByteArrayOutputStream();
......
195 92
		//the names of all superclasses of the entity
196 93
		jg.writeArrayFieldStart("tags");
197 94
		Iterator<String> classNames = resourceReader.getRDFClassNames(res);
198
		while(classNames.hasNext()) {
95
		while (classNames.hasNext()) {
199 96
			jg.writeStartObject();
200 97
			jg.writeStringField("name", classNames.next());
201 98
			jg.writeEndObject();
......
213 110
		jg.writeEndArray();
214 111

  
215 112
		jg.writeArrayFieldStart("extras");
216
		addExtra(jg,"system:type", CRM.E7_Activity.getLocalName());
113
		addExtra(jg, "system:type", E7_Activity.getLocalName());
217 114
		//specific class
218
		addExtra(jg,"instance of", resourceReader.findSpecificType(res, CRM.E7_Activity).getLocalName());
219
		if(res.getURI().startsWith("http://parthenos.d4science.org"))
220
			addExtra(jg,"Parthenos URL", res.getURI());
221
		addExtra(jg,"competence", res.getURI());
115
		addExtra(jg, "instance of", resourceReader.findSpecificType(res, E7_Activity).getLocalName());
116
		if (res.getURI().startsWith("http://parthenos.d4science.org"))
117
			addExtra(jg, "Parthenos URL", res.getURI());
118
		addExtra(jg, "competence", res.getURI());
222 119
		//TODO: from which linked E39_Actors? Should be the same as for groups above
223
		addExtra(jg,"provided by", "");
120
		addExtra(jg, "provided by", "");
224 121
		//TODO: get begin/end of operation from PP42_has_declarative_time
225
		addExtra(jg,"declared begin/end of operation", "");
122
		addExtra(jg, "declared begin/end of operation", "");
226 123
		//TODO: where to get it?
227
		addExtra(jg,"last confirmation", "");
124
		addExtra(jg, "last confirmation", "");
228 125
		//TODO: where to get it?
229
		addExtra(jg,"date of registration", "");
230
		addExtra(jg,"availability", resourceReader.getAvailability(res));
231
		addExtra(jg,"condition of use", resourceReader.getConditionOfUse(res));
126
		addExtra(jg, "date of registration", "");
127
		addExtra(jg, "availability", resourceReader.getAvailability(res));
128
		addExtra(jg, "condition of use", resourceReader.getConditionOfUse(res));
232 129
		//TODO: where to get the contact person
233 130
		//Resource contactPerson = null;
234 131
		//jg.writeStringField("contact person", resourceReader.getTitle(contactPerson));
235 132
		//TODO: where to get it?
236
		addExtra(jg,"activity type", "");
133
		addExtra(jg, "activity type", "");
237 134
		// can't add multiple fields with same name, hence skipping
238 135
		//jg.writeStringField("hosts", "");
239 136
		//TODO: where to get it?
240
		addExtra(jg,"online access point", "");
137
		addExtra(jg, "online access point", "");
241 138
		//TODO: where to get it?
242
		addExtra(jg,"authorization", "");
139
		addExtra(jg, "authorization", "");
243 140
		//TODO: where to get it?
244
		addExtra(jg,"protocol", "");
141
		addExtra(jg, "protocol", "");
245 142
		//can't add multiple fields with same name, hence skipping
246 143
		//addExtra(jg,"curates", "");
247 144
		//TODO: where to get it?
248
		addExtra(jg,"runs on request", "");
145
		addExtra(jg, "runs on request", "");
249 146
		//TODO: where to get it?
250
		addExtra(jg,"delivers on request", "");
147
		addExtra(jg, "delivers on request", "");
251 148
		jg.writeEndArray(); //end extras
252 149

  
253 150
		jg.writeEndObject();
254 151
		jg.close();
255 152
		return out.toString("UTF-8");
256 153
	}
154
	protected String getJsonForActor(final Resource res, final String resNameForCatalogue) throws IOException {
155
		JsonFactory jsonFactory = new JsonFactory();
156
		final ByteArrayOutputStream out = new ByteArrayOutputStream();
157
		BufferedOutputStream bos = new BufferedOutputStream(out);
158
		JsonGenerator jg = jsonFactory.createGenerator(bos, JsonEncoding.UTF8);
159
		jg.writeStartObject();
160
		writeCommonFields(jg, res, resNameForCatalogue);
257 161

  
162
		jg.writeArrayFieldStart("extras");
163
		addExtra(jg, "system:type", E39_Actor.getLocalName());
164
		//specific class
165
		addExtra(jg, "instance of", resourceReader.findSpecificType(res, E39_Actor).getLocalName());
166
		if (res.getURI().startsWith("http://parthenos.d4science.org"))
167
			addExtra(jg, "Parthenos URL", res.getURI());
168
		jg.writeEndArray();
258 169

  
259
	private int doRegister(final String json, final String objIdentifier, final String catName) throws ParthenosPublisherException {
260
		//TODO: implement me
261
		log.debug(String.format("Catalogue --> Registering %s (from %s) : %s", catName, objIdentifier, json));
262
		//if resCatName is already registered --> delete it and then saves it again
263
		try {
264
			URI uri = new URIBuilder(getBaseURL()).build();
265
			HttpGet req = new HttpGet(uri);
266
			setHeaders(req);
267
			System.out.println(req.getURI());
268
			return 0;
269
		}catch(Throwable t){
270
			throw new ParthenosPublisherException(t);
170
		jg.writeEndObject();
171
		jg.close();
172
		return out.toString("UTF-8");
173
	}
174
	protected String getJsonForThing(final Resource res, final String resNameForCatalogue) throws IOException {
175
		JsonFactory jsonFactory = new JsonFactory();
176
		final ByteArrayOutputStream out = new ByteArrayOutputStream();
177
		BufferedOutputStream bos = new BufferedOutputStream(out);
178
		JsonGenerator jg = jsonFactory.createGenerator(bos, JsonEncoding.UTF8);
179
		jg.writeStartObject();
180
		writeCommonFields(jg, res, resNameForCatalogue);
181

  
182
		jg.writeArrayFieldStart("extras");
183
		addExtra(jg, "system:type", E70_Thing.getLocalName());
184
		//specific class
185
		addExtra(jg, "instance of", resourceReader.findSpecificType(res, E70_Thing).getLocalName());
186
		if (res.getURI().startsWith("http://parthenos.d4science.org")) {
187
			addExtra(jg, "Parthenos URL", res.getURI());
271 188
		}
189
		//TODO: things include digital objects, software, datasets, schema. Guess we should know what to add here.
190
		jg.writeEndArray();
191

  
192
		jg.writeEndObject();
193
		jg.close();
194
		return out.toString("UTF-8");
272 195
	}
273 196

  
274
	private void setHeaders(final HttpRequestBase req){
275
		req.setHeader("Content-type", "application/json");
276
		req.setHeader("Accept","application/json");
277
		req.setHeader("gcube-token", getApplicationToken() );
197
	protected void addExtra(final JsonGenerator jg, final String key, final String value) throws IOException {
198
		jg.writeStartObject();
199
		jg.writeStringField("key", key);
200
		jg.writeStringField("value", value);
201
		jg.writeEndObject();
278 202
	}
279 203

  
280
	protected InfModel loadBaseModel() {
281
		return ModelFactory.createRDFSModel(baseModel);
204
	protected void writeCommonFields(final JsonGenerator jg, final Resource res, final String resNameForCatalogue) throws IOException {
205
		//end of URI
206
		jg.writeStringField("name", resNameForCatalogue);
207
		//default license
208
		jg.writeStringField("license_id", "notspecified");
209
		String title = resourceReader.getTitle(res);
210
		if (StringUtils.isBlank(title))
211
			title = resNameForCatalogue;
212
		jg.writeStringField("title", title);
213
		//description
214
		jg.writeStringField("notes", resourceReader.getDescription(res));
282 215
	}
283 216

  
284
	public OntModel getBaseModel() {
285
		return baseModel;
217
	/**
218
	 * @param json
219
	 * @param resCatName
220
	 * @return the uuid assigned by the catalogue to the registered resource
221
	 * @throws ParthenosPublisherException
222
	 */
223
	private String doRegister(final String json, final String resCatName) throws ParthenosPublisherException {
224
		//TODO: implement me
225
		log.debug(String.format("Catalogue --> Registering %s : %s", resCatName, json));
226
		//if resCatName is already registered --> delete it and then saves it again
227
		HttpEntity<String> entity = new HttpEntity<String>(json, getHeaders());
228
		try {
229
			URI uri = new URIBuilder(getBaseURL()).addParameter("gcube-token", getApplicationToken()).build();
230
			ResponseEntity<String> res = jrrRestTemplate.<String>exchange(uri, HttpMethod.POST, entity, String.class);
231
			//TODO: I guess response is a json, need to check the format, parse it to get the uuid of the registered resource
232
			String uuid = res.getBody();
233
			return uuid;
234
		} catch (Throwable t) {
235
			throw new ParthenosPublisherException(t);
236
		}
286 237
	}
287 238

  
288
	public void setBaseModel(final OntModel baseModel) {
289
		this.baseModel = baseModel;
239
	private HttpHeaders getHeaders() {
240
		HttpHeaders headers = new HttpHeaders();
241
		headers.setContentType(MediaType.APPLICATION_JSON);
242
		headers.setAccept(Lists.newArrayList(MediaType.APPLICATION_JSON));
243
		//headers.set("gcube-token", getApplicationToken() );
244
		return headers;
290 245
	}
291 246

  
247

  
292 248
	public String getBaseURL() {
293 249
		return baseURL;
294 250
	}

Also available in: Unified diff