1
|
package eu.dnetlib.parthenos.virtuoso;
|
2
|
|
3
|
import eu.dnetlib.parthenos.publisher.ParthenosPublisherException;
|
4
|
import eu.dnetlib.parthenos.rdf.RecordParserHelper;
|
5
|
import org.apache.commons.io.IOUtils;
|
6
|
import org.apache.commons.lang3.StringUtils;
|
7
|
import org.apache.commons.logging.Log;
|
8
|
import org.apache.commons.logging.LogFactory;
|
9
|
import org.apache.jena.datatypes.xsd.XSDDatatype;
|
10
|
import org.apache.jena.rdf.model.*;
|
11
|
import virtuoso.jena.driver.VirtModel;
|
12
|
|
13
|
/**
|
14
|
* Created by Alessia Bardi on 12/07/2017.
|
15
|
*
|
16
|
* @author Alessia Bardi
|
17
|
*/
|
18
|
public class VirtuosoClient {
|
19
|
|
20
|
private static final Log log = LogFactory.getLog(VirtuosoClient.class);
|
21
|
|
22
|
public static final String PROVENANCE_GRAPH = "provenance";
|
23
|
public static Property IS_API_OF = ResourceFactory.createProperty("dnet", "isApiOf");
|
24
|
public static Property COLL_FROM = ResourceFactory.createProperty("dnet", "collectedFrom");
|
25
|
public static Property COLL_IN_DATE = ResourceFactory.createProperty("dnet", "collectedInDate");
|
26
|
public static Property TRANS_IN_DATE = ResourceFactory.createProperty("dnet", "transformedInDate");
|
27
|
|
28
|
private RecordParserHelper recordParserHelper;
|
29
|
|
30
|
private String connectionString;
|
31
|
private String username;
|
32
|
private String password;
|
33
|
private String defaultBaseURI;
|
34
|
|
35
|
protected VirtuosoClient(final String connectionString,
|
36
|
final String username,
|
37
|
final String password,
|
38
|
final RecordParserHelper recordParserHelper,
|
39
|
final String defaultBaseURI) {
|
40
|
this.connectionString = connectionString;
|
41
|
this.username = username;
|
42
|
this.password = password;
|
43
|
this.recordParserHelper = recordParserHelper;
|
44
|
this.defaultBaseURI = defaultBaseURI;
|
45
|
}
|
46
|
|
47
|
//TODO: exploit new method eu.dnetlib.parthenos.publisher.SaxonHelper.Helper.parseXML() to avoid re-parsing the full record.
|
48
|
|
49
|
public long feed(final String record) throws ParthenosPublisherException{
|
50
|
Model md = null ;
|
51
|
try {
|
52
|
if (StringUtils.isBlank(record)) {
|
53
|
log.warn("Got empty record");
|
54
|
return 0;
|
55
|
}
|
56
|
String objIdentifier = recordParserHelper.getObjIdentifier(record);
|
57
|
if (StringUtils.isBlank(objIdentifier)) {
|
58
|
log.warn("Got record with no objIdentifier -- skipping");
|
59
|
return 0;
|
60
|
}
|
61
|
String rdfBlock = recordParserHelper.getRDF(record);
|
62
|
if (StringUtils.isBlank(rdfBlock)) {
|
63
|
log.warn("Missing rdf:RDF in record with objIdentifier " + objIdentifier + " all triples in that named graph will be deleted");
|
64
|
}
|
65
|
String collectionDate = recordParserHelper.getCollectionDate(record);
|
66
|
String transformationDate = recordParserHelper.getTransformationDate(record);
|
67
|
String datasource = recordParserHelper.getDatasourceName(record);
|
68
|
String dsInterface = recordParserHelper.getDatasourceApi(record);
|
69
|
|
70
|
String namedGraph = getRecordDefaultURI(objIdentifier, dsInterface);
|
71
|
log.debug("Trying to open the database model " + namedGraph+", connection string "+getConnectionString());
|
72
|
|
73
|
md = VirtModel.openDatabaseModel(namedGraph, getConnectionString(), getUsername(), getPassword());
|
74
|
log.debug("Opened virtuoso model for graph " + namedGraph);
|
75
|
md.removeAll();
|
76
|
log.debug("Removed all triples from graph " + namedGraph);
|
77
|
md.read(IOUtils.toInputStream(rdfBlock, "UTF-8"), getDefaultBaseURI());
|
78
|
long size = md.size();
|
79
|
log.debug("Graph " + namedGraph + " now has " + size + " triples");
|
80
|
|
81
|
long ntriples = feedProvenance(namedGraph, collectionDate, transformationDate, datasource, dsInterface);
|
82
|
log.debug("provenance graph for " + namedGraph + " updated with " + ntriples + " triples");
|
83
|
md.close();
|
84
|
return size;
|
85
|
}catch(Throwable e){
|
86
|
if (md != null && !md.isClosed()) md.close();
|
87
|
log.error(e);
|
88
|
throw new ParthenosPublisherException(e);
|
89
|
}
|
90
|
}
|
91
|
|
92
|
long feedProvenance(final String namedGraphURI, final String collectionDate, final String transformationDate, final String datasource, final String api) {
|
93
|
Model md = VirtModel.openDatabaseModel(PROVENANCE_GRAPH, getConnectionString(), getUsername(), getPassword());
|
94
|
|
95
|
//TODO: use prov-o instead: https://www.w3.org/TR/prov-o/#description
|
96
|
Resource rApi = ResourceFactory.createResource(defaultBaseURI + api);
|
97
|
Resource r = ResourceFactory.createResource(namedGraphURI);
|
98
|
Statement stmApi =
|
99
|
ResourceFactory.createStatement(rApi, IS_API_OF, ResourceFactory.createPlainLiteral(datasource));
|
100
|
Statement stmCollFrom =
|
101
|
ResourceFactory.createStatement(r, COLL_FROM, rApi);
|
102
|
Statement stmCollDate = ResourceFactory
|
103
|
.createStatement(r, COLL_IN_DATE, ResourceFactory.createTypedLiteral(collectionDate, XSDDatatype.XSDdateTime));
|
104
|
Statement stmTransDate = ResourceFactory
|
105
|
.createStatement(r, TRANS_IN_DATE, ResourceFactory.createTypedLiteral(transformationDate, XSDDatatype.XSDdateTime));
|
106
|
|
107
|
//let's remove previous provenance statements for this resource:
|
108
|
md.removeAll(r, null, null);
|
109
|
//and add the new ones
|
110
|
md.add(stmApi).add(stmCollFrom).add(stmCollDate).add(stmTransDate);
|
111
|
md.close();
|
112
|
return 3;
|
113
|
}
|
114
|
|
115
|
public long feed(final Iterable<String> records) throws ParthenosPublisherException {
|
116
|
//TODO: can we do it in parallel? if all records have different objIdentifier it is safe, and this must be the case anyway, because the source of records is a D-Net mdstore.
|
117
|
long count = 0;
|
118
|
for (String r : records) count += this.feed(r);
|
119
|
return count;
|
120
|
}
|
121
|
|
122
|
/**
|
123
|
* Delete all triples in named graphs collected from the given api
|
124
|
* @param api the id of the API
|
125
|
* @return the number of triples deleted from the named graphs associated to the given api
|
126
|
*/
|
127
|
public long drop(final String api){
|
128
|
Model prov = VirtModel.openDatabaseModel(PROVENANCE_GRAPH, getConnectionString(), getUsername(), getPassword());
|
129
|
//look for all named graphs associated to the api
|
130
|
Resource rApi = ResourceFactory.createResource(defaultBaseURI + api);
|
131
|
long deletedTriples = 0;
|
132
|
final ResIterator resIterator = prov.listSubjectsWithProperty(COLL_FROM, rApi);
|
133
|
while (resIterator.hasNext()) {
|
134
|
Resource namedGraphURI = resIterator.nextResource();
|
135
|
//delete all triples belonging to the r named graph
|
136
|
deletedTriples += dropNamedGraph(namedGraphURI.getURI());
|
137
|
//delete the named graph from the provenance graph
|
138
|
prov.removeAll(namedGraphURI, null, null);
|
139
|
}
|
140
|
//delete the api from the provenance graph
|
141
|
prov.removeAll(null, null, rApi);
|
142
|
prov.removeAll(rApi, null, null);
|
143
|
prov.close();
|
144
|
return deletedTriples;
|
145
|
}
|
146
|
|
147
|
private long dropNamedGraph(String namedGraphURI){
|
148
|
Model namedGraph = VirtModel.openDatabaseModel(namedGraphURI, getConnectionString(), getUsername(), getPassword());
|
149
|
long deletedTriples = namedGraph.size();
|
150
|
namedGraph.removeAll();
|
151
|
namedGraph.close();
|
152
|
return deletedTriples;
|
153
|
}
|
154
|
|
155
|
private String getRecordDefaultURI(final String objIdentifier, final String datasourceApi) {
|
156
|
return defaultBaseURI + datasourceApi + "/" + objIdentifier;
|
157
|
}
|
158
|
|
159
|
public String getConnectionString() {
|
160
|
return connectionString;
|
161
|
}
|
162
|
|
163
|
public String getUsername() {
|
164
|
return username;
|
165
|
}
|
166
|
|
167
|
public String getPassword() {
|
168
|
return password;
|
169
|
}
|
170
|
|
171
|
public String getDefaultBaseURI() {
|
172
|
return defaultBaseURI;
|
173
|
}
|
174
|
|
175
|
public RecordParserHelper getRecordParserHelper() {
|
176
|
return recordParserHelper;
|
177
|
}
|
178
|
|
179
|
public void setRecordParserHelper(final RecordParserHelper recordParserHelper) {
|
180
|
this.recordParserHelper = recordParserHelper;
|
181
|
}
|
182
|
|
183
|
public void setConnectionString(final String connectionString) {
|
184
|
this.connectionString = connectionString;
|
185
|
}
|
186
|
|
187
|
public void setUsername(final String username) {
|
188
|
this.username = username;
|
189
|
}
|
190
|
|
191
|
public void setPassword(final String password) {
|
192
|
this.password = password;
|
193
|
}
|
194
|
|
195
|
public void setDefaultBaseURI(final String defaultBaseURI) {
|
196
|
this.defaultBaseURI = defaultBaseURI;
|
197
|
}
|
198
|
}
|