Revision 48553
Added by Alessia Bardi about 7 years ago
| modules/dnet-parthenos-publisher/trunk/test/main/java/eu/dnetlib/parthenos/virtuoso/VirtuosoClientTest.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.parthenos.virtuoso; |
|
| 2 |
|
|
| 3 |
import eu.dnetlib.miscutils.functional.xml.SaxonHelper; |
|
| 4 |
import net.sf.saxon.s9api.SaxonApiException; |
|
| 5 |
import org.junit.Assert; |
|
| 6 |
import org.junit.Before; |
|
| 7 |
import org.junit.Ignore; |
|
| 8 |
import org.junit.Test; |
|
| 9 |
|
|
| 10 |
/** |
|
| 11 |
* Created by Alessia Bardi on 12/07/2017. |
|
| 12 |
* |
|
| 13 |
* @author Alessia Bardi |
|
| 14 |
*/ |
|
| 15 |
@Ignore |
|
| 16 |
public class VirtuosoClientTest {
|
|
| 17 |
|
|
| 18 |
private String testGraph = "virt:test"; |
|
| 19 |
private String connectionString = "jdbc:virtuoso://localhost:1111"; |
|
| 20 |
private String testUser = "dba"; |
|
| 21 |
private String testPwd = "dba"; |
|
| 22 |
private String defaultURIBaseURl = "http://test/"; |
|
| 23 |
|
|
| 24 |
private VirtuosoClient client; |
|
| 25 |
|
|
| 26 |
@Before |
|
| 27 |
public void prepare() throws SaxonApiException {
|
|
| 28 |
client = new VirtuosoClient(connectionString, testUser, testPwd, new SaxonHelper(), defaultURIBaseURl); |
|
| 29 |
} |
|
| 30 |
|
|
| 31 |
@Test |
|
| 32 |
public void testFeedProvenance(){
|
|
| 33 |
long res = client.feedProvenance("ariadne_____::0002b8b77eb2d6c62f61c6e1a1ce6891", "2017-05-14T12:06:38.152+02:00" , "2017-05-14T12:11:55.573", "ARIADNE");
|
|
| 34 |
Assert.assertEquals(3, res); |
|
| 35 |
} |
|
| 36 |
} |
|
| modules/dnet-parthenos-publisher/trunk/test/main/java/eu/dnetlib/parthenos/virtuoso/VirtuosoTest.java | ||
|---|---|---|
| 6 | 6 |
import org.apache.jena.graph.NodeFactory; |
| 7 | 7 |
import org.apache.jena.graph.Triple; |
| 8 | 8 |
import org.apache.jena.rdf.model.Model; |
| 9 |
import org.apache.jena.rdf.model.ResourceFactory; |
|
| 9 | 10 |
import org.junit.Assert; |
| 10 | 11 |
import org.junit.Before; |
| 11 | 12 |
import org.junit.Ignore; |
| ... | ... | |
| 72 | 73 |
long size = md.size(); |
| 73 | 74 |
System.out.println("==Inserted " + size + "triples==");
|
| 74 | 75 |
|
| 75 |
VirtGraph graph = new VirtGraph(testGraph, "jdbc:virtuoso://localhost:1111", "dba", "dba");
|
|
| 76 |
VirtGraph graph = new VirtGraph(testGraph, connectionString, testUser, testPwd);
|
|
| 76 | 77 |
assertNotNull(graph); |
| 77 | 78 |
Assert.assertFalse(graph.isEmpty()); |
| 78 | 79 |
|
| ... | ... | |
| 81 | 82 |
|
| 82 | 83 |
/* |
| 83 | 84 |
If we want to be able to update, we need to store the triples in different named graph. This way we can delete the old triples and add the new ones. |
| 84 |
And then we can add triples about the graph (e.g. provenance, last update time, whatever) in another named graph (e.g. 'provenance'), where all the info about the named graphs are available. |
|
| 85 |
And then we can add triples about the graph (e.g. provenance, last update time, whatever) in another named graph (e.g. 'provenance'), |
|
| 86 |
where all the info about the named graphs are available. |
|
| 85 | 87 |
*/ |
| 86 | 88 |
@Test |
| 87 | 89 |
public void testUpdate() {
|
| ... | ... | |
| 102 | 104 |
|
| 103 | 105 |
} |
| 104 | 106 |
|
| 107 |
|
|
| 108 |
@Test |
|
| 109 |
public void testRemoveAllWildcards() {
|
|
| 110 |
VirtGraph graph = new VirtGraph(testGraph, connectionString, testUser, testPwd); |
|
| 111 |
assertNotNull(graph); |
|
| 112 |
graph.clear(); |
|
| 113 |
Assert.assertTrue(graph.isEmpty()); |
|
| 114 |
Triple t1 = new Triple(NodeFactory.createURI("http://test/1"), NodeFactory.createURI("http://test/is_friend_of"),
|
|
| 115 |
NodeFactory.createURI("http://test/KevinBacon"));
|
|
| 116 |
Triple t2 = new Triple(NodeFactory.createURI("http://test/1"), NodeFactory.createURI("http://test/is_friend_of"),
|
|
| 117 |
NodeFactory.createURI("http://test/BonoVox"));
|
|
| 118 |
Triple t3 = new Triple(NodeFactory.createURI("http://test/1"), NodeFactory.createURI("http://test/is_enemy_of"),
|
|
| 119 |
NodeFactory.createURI("http://test/Spiderman"));
|
|
| 120 |
graph.performAdd(t1); |
|
| 121 |
graph.performAdd(t2); |
|
| 122 |
graph.performAdd(t3); |
|
| 123 |
assertEquals(3, graph.getCount()); |
|
| 124 |
|
|
| 125 |
Model md = VirtModel.openDatabaseModel(testGraph, connectionString, testUser, testPwd); |
|
| 126 |
assertEquals(3, md.size()); |
|
| 127 |
md.removeAll(ResourceFactory.createResource("http://test/1"), ResourceFactory.createProperty("http://test/is_enemy_of"), null);
|
|
| 128 |
assertEquals(2, md.size()); |
|
| 129 |
} |
|
| 130 |
|
|
| 131 |
|
|
| 105 | 132 |
private InputStream getResourceAsStream(final String classpath) {
|
| 106 | 133 |
try {
|
| 107 | 134 |
final ClassPathResource resource = new ClassPathResource(classpath); |
| modules/dnet-parthenos-publisher/trunk/dnet-parthenos-publisher.iml | ||
|---|---|---|
| 1 | 1 |
<?xml version="1.0" encoding="UTF-8"?> |
| 2 | 2 |
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4"> |
| 3 |
<component name="FacetManager"> |
|
| 4 |
<facet type="Spring" name="Spring"> |
|
| 5 |
<configuration /> |
|
| 6 |
</facet> |
|
| 7 |
</component> |
|
| 3 | 8 |
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8"> |
| 4 | 9 |
<output url="file://$MODULE_DIR$/target/classes" /> |
| 5 | 10 |
<output-test url="file://$MODULE_DIR$/target/test-classes" /> |
| ... | ... | |
| 12 | 17 |
</content> |
| 13 | 18 |
<orderEntry type="inheritedJdk" /> |
| 14 | 19 |
<orderEntry type="sourceFolder" forTests="false" /> |
| 20 |
<orderEntry type="module" module-name="dnet-core-components" /> |
|
| 21 |
<orderEntry type="library" name="Maven: org.springframework:spring-webmvc:4.2.5.RELEASE" level="project" /> |
|
| 22 |
<orderEntry type="library" name="Maven: org.springframework:spring-expression:4.2.5.RELEASE" level="project" /> |
|
| 23 |
<orderEntry type="library" name="Maven: org.springframework:spring-web:4.2.5.RELEASE" level="project" /> |
|
| 24 |
<orderEntry type="library" name="Maven: org.quartz-scheduler:quartz:2.2.2" level="project" /> |
|
| 25 |
<orderEntry type="library" name="Maven: c3p0:c3p0:0.9.1.1" level="project" /> |
|
| 26 |
<orderEntry type="library" name="Maven: org.springframework:spring-context:4.2.5.RELEASE" level="project" /> |
|
| 27 |
<orderEntry type="library" name="Maven: org.springframework:spring-aop:4.2.5.RELEASE" level="project" /> |
|
| 28 |
<orderEntry type="library" name="Maven: aopalliance:aopalliance:1.0" level="project" /> |
|
| 29 |
<orderEntry type="library" name="Maven: org.springframework:spring-beans:4.2.5.RELEASE" level="project" /> |
|
| 30 |
<orderEntry type="library" name="Maven: org.springframework:spring-context-support:4.2.5.RELEASE" level="project" /> |
|
| 31 |
<orderEntry type="library" name="Maven: com.google.guava:guava:18.0" level="project" /> |
|
| 32 |
<orderEntry type="library" name="Maven: net.sf.ehcache:ehcache-core:2.6.2" level="project" /> |
|
| 33 |
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-queryparser:5.5.0" level="project" /> |
|
| 34 |
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-core:5.5.0" level="project" /> |
|
| 35 |
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-queries:5.5.0" level="project" /> |
|
| 36 |
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-sandbox:5.5.0" level="project" /> |
|
| 37 |
<orderEntry type="library" name="Maven: org.z3950.zing:cql-java:1.7" level="project" /> |
|
| 38 |
<orderEntry type="library" name="Maven: org.antlr:stringtemplate:3.2" level="project" /> |
|
| 39 |
<orderEntry type="library" name="Maven: org.antlr:antlr:2.7.7" level="project" /> |
|
| 40 |
<orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.4" level="project" /> |
|
| 41 |
<orderEntry type="library" name="Maven: commons-io:commons-io:2.4" level="project" /> |
|
| 42 |
<orderEntry type="library" name="Maven: com.google.code.gson:gson:2.6.2" level="project" /> |
|
| 43 |
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.4.1" level="project" /> |
|
| 44 |
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.4.1" level="project" /> |
|
| 45 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-core:3.1.5" level="project" /> |
|
| 46 |
<orderEntry type="library" name="Maven: org.codehaus.woodstox:woodstox-core-asl:4.4.1" level="project" /> |
|
| 47 |
<orderEntry type="library" name="Maven: org.codehaus.woodstox:stax2-api:3.1.4" level="project" /> |
|
| 48 |
<orderEntry type="library" name="Maven: org.apache.ws.xmlschema:xmlschema-core:2.2.1" level="project" /> |
|
| 49 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-frontend-jaxws:3.1.5" level="project" /> |
|
| 50 |
<orderEntry type="library" name="Maven: xml-resolver:xml-resolver:1.2" level="project" /> |
|
| 51 |
<orderEntry type="library" name="Maven: org.ow2.asm:asm:5.0.4" level="project" /> |
|
| 52 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-bindings-soap:3.1.5" level="project" /> |
|
| 53 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-wsdl:3.1.5" level="project" /> |
|
| 54 |
<orderEntry type="library" name="Maven: wsdl4j:wsdl4j:1.6.3" level="project" /> |
|
| 55 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-databinding-jaxb:3.1.5" level="project" /> |
|
| 56 |
<orderEntry type="library" name="Maven: com.sun.xml.bind:jaxb-impl:2.2.11" level="project" /> |
|
| 57 |
<orderEntry type="library" name="Maven: com.sun.xml.bind:jaxb-core:2.2.11" level="project" /> |
|
| 58 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-bindings-xml:3.1.5" level="project" /> |
|
| 59 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-frontend-simple:3.1.5" level="project" /> |
|
| 60 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-ws-addr:3.1.5" level="project" /> |
|
| 61 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-ws-policy:3.1.5" level="project" /> |
|
| 62 |
<orderEntry type="library" name="Maven: org.apache.neethi:neethi:3.0.3" level="project" /> |
|
| 63 |
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.8" level="project" /> |
|
| 64 |
<orderEntry type="library" name="Maven: net.sf.saxon:Saxon-HE:9.5.1-5" level="project" /> |
|
| 65 |
<orderEntry type="library" name="Maven: jaxen:jaxen:1.1.6" level="project" /> |
|
| 66 |
<orderEntry type="library" name="Maven: dom4j:dom4j:1.6.1" level="project" /> |
|
| 67 |
<orderEntry type="library" name="Maven: com.ximpleware:vtd-xml:2.13.2" level="project" /> |
|
| 68 |
<orderEntry type="library" name="Maven: com.mycila:xmltool:3.3" level="project" /> |
|
| 69 |
<orderEntry type="library" name="Maven: org.mongodb:mongo-java-driver:3.4.2" level="project" /> |
|
| 70 |
<orderEntry type="library" name="Maven: joda-time:joda-time:2.9.9" level="project" /> |
|
| 71 |
<orderEntry type="library" name="Maven: org.springframework:spring-test:4.2.5.RELEASE" level="project" /> |
|
| 15 | 72 |
<orderEntry type="library" name="Maven: virtuoso:jena-driver:3.0" level="project" /> |
| 16 | 73 |
<orderEntry type="library" name="Maven: openlink:virtuoso-jdbc:4.0" level="project" /> |
| 17 |
<orderEntry type="library" name="Maven: org.apache.jena:apache-jena:3.3.0" level="project" /> |
|
| 18 | 74 |
<orderEntry type="library" name="Maven: org.apache.jena:jena-arq:3.3.0" level="project" /> |
| 19 | 75 |
<orderEntry type="library" name="Maven: org.apache.jena:jena-shaded-guava:3.3.0" level="project" /> |
| 20 |
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.5.2" level="project" /> |
|
| 21 |
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.4.4" level="project" /> |
|
| 22 |
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.9" level="project" /> |
|
| 23 | 76 |
<orderEntry type="library" name="Maven: com.github.jsonld-java:jsonld-java:0.9.0" level="project" /> |
| 24 | 77 |
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.7.4" level="project" /> |
| 25 | 78 |
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.7.4" level="project" /> |
| 26 | 79 |
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.7.0" level="project" /> |
| 27 |
<orderEntry type="library" name="Maven: commons-io:commons-io:2.5" level="project" /> |
|
| 28 | 80 |
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient-cache:4.5.2" level="project" /> |
| 29 | 81 |
<orderEntry type="library" name="Maven: org.apache.thrift:libthrift:0.9.3" level="project" /> |
| 30 | 82 |
<orderEntry type="library" name="Maven: org.slf4j:jcl-over-slf4j:1.7.21" level="project" /> |
| 31 | 83 |
<orderEntry type="library" name="Maven: org.apache.commons:commons-csv:1.3" level="project" /> |
| 32 |
<orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.4" level="project" /> |
|
| 33 | 84 |
<orderEntry type="library" name="Maven: org.apache.jena:jena-tdb:3.3.0" level="project" /> |
| 34 | 85 |
<orderEntry type="library" name="Maven: org.apache.jena:jena-rdfconnection:3.3.0" level="project" /> |
| 35 | 86 |
<orderEntry type="library" name="Maven: org.apache.jena:jena-cmds:3.3.0" level="project" /> |
| ... | ... | |
| 45 | 96 |
<orderEntry type="library" name="Maven: junit:junit:4.12" level="project" /> |
| 46 | 97 |
<orderEntry type="library" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" /> |
| 47 | 98 |
<orderEntry type="library" scope="TEST" name="Maven: org.springframework:spring-core:4.2.5.RELEASE" level="project" /> |
| 48 |
<orderEntry type="library" scope="TEST" name="Maven: commons-logging:commons-logging:1.2" level="project" />
|
|
| 99 |
<orderEntry type="library" name="Maven: commons-logging:commons-logging:1.2" level="project" /> |
|
| 49 | 100 |
<orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" /> |
| 50 | 101 |
</component> |
| 51 | 102 |
</module> |
| modules/dnet-parthenos-publisher/trunk/src/main/java/eu/dnetlib/parthenos/virtuoso/VirtuosoClientFactory.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.parthenos.virtuoso; |
|
| 2 |
|
|
| 3 |
import eu.dnetlib.miscutils.functional.xml.SaxonHelper; |
|
| 4 |
import net.sf.saxon.s9api.SaxonApiException; |
|
| 5 |
import org.springframework.beans.factory.annotation.Autowired; |
|
| 6 |
|
|
| 7 |
/** |
|
| 8 |
* Created by Alessia Bardi on 12/07/2017. |
|
| 9 |
* |
|
| 10 |
* @author Alessia Bardi |
|
| 11 |
*/ |
|
| 12 |
public class VirtuosoClientFactory {
|
|
| 13 |
|
|
| 14 |
private String connectionString; |
|
| 15 |
private String username; |
|
| 16 |
private String password; |
|
| 17 |
private String defaultBaseURI; |
|
| 18 |
|
|
| 19 |
@Autowired |
|
| 20 |
private SaxonHelper saxonHelper; |
|
| 21 |
|
|
| 22 |
public VirtuosoClient getVirtuosoClient() throws SaxonApiException {
|
|
| 23 |
return new VirtuosoClient(connectionString, username, password, getSaxonHelper(), defaultBaseURI); |
|
| 24 |
} |
|
| 25 |
|
|
| 26 |
public String getConnectionString() {
|
|
| 27 |
return connectionString; |
|
| 28 |
} |
|
| 29 |
|
|
| 30 |
public void setConnectionString(final String connectionString) {
|
|
| 31 |
this.connectionString = connectionString; |
|
| 32 |
} |
|
| 33 |
|
|
| 34 |
public String getUsername() {
|
|
| 35 |
return username; |
|
| 36 |
} |
|
| 37 |
|
|
| 38 |
public void setUsername(final String username) {
|
|
| 39 |
this.username = username; |
|
| 40 |
} |
|
| 41 |
|
|
| 42 |
public String getPassword() {
|
|
| 43 |
return password; |
|
| 44 |
} |
|
| 45 |
|
|
| 46 |
public void setPassword(final String password) {
|
|
| 47 |
this.password = password; |
|
| 48 |
} |
|
| 49 |
|
|
| 50 |
public SaxonHelper getSaxonHelper() {
|
|
| 51 |
return saxonHelper; |
|
| 52 |
} |
|
| 53 |
|
|
| 54 |
public void setSaxonHelper(final SaxonHelper saxonHelper) {
|
|
| 55 |
this.saxonHelper = saxonHelper; |
|
| 56 |
} |
|
| 57 |
|
|
| 58 |
public String getDefaultBaseURI() {
|
|
| 59 |
return defaultBaseURI; |
|
| 60 |
} |
|
| 61 |
|
|
| 62 |
public void setDefaultBaseURI(final String defaultBaseURI) {
|
|
| 63 |
this.defaultBaseURI = defaultBaseURI; |
|
| 64 |
} |
|
| 65 |
} |
|
| modules/dnet-parthenos-publisher/trunk/src/main/java/eu/dnetlib/parthenos/virtuoso/VirtuosoClient.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.parthenos.virtuoso; |
|
| 2 |
|
|
| 3 |
import java.util.Map; |
|
| 4 |
|
|
| 5 |
import com.google.common.collect.Maps; |
|
| 6 |
import eu.dnetlib.miscutils.functional.xml.SaxonHelper; |
|
| 7 |
import net.sf.saxon.s9api.SaxonApiException; |
|
| 8 |
import net.sf.saxon.s9api.Serializer.Property; |
|
| 9 |
import net.sf.saxon.s9api.XPathSelector; |
|
| 10 |
import org.apache.commons.io.IOUtils; |
|
| 11 |
import org.apache.commons.lang3.StringUtils; |
|
| 12 |
import org.apache.commons.logging.Log; |
|
| 13 |
import org.apache.commons.logging.LogFactory; |
|
| 14 |
import org.apache.jena.rdf.model.Model; |
|
| 15 |
import org.apache.jena.rdf.model.Resource; |
|
| 16 |
import org.apache.jena.rdf.model.ResourceFactory; |
|
| 17 |
import org.apache.jena.rdf.model.Statement; |
|
| 18 |
import virtuoso.jena.driver.VirtModel; |
|
| 19 |
|
|
| 20 |
/** |
|
| 21 |
* Created by Alessia Bardi on 12/07/2017. |
|
| 22 |
* |
|
| 23 |
* @author Alessia Bardi |
|
| 24 |
*/ |
|
| 25 |
public class VirtuosoClient {
|
|
| 26 |
|
|
| 27 |
private static final Log log = LogFactory.getLog(VirtuosoClient.class); |
|
| 28 |
|
|
| 29 |
private static final String OAI_NAMESPACE_URI = "http://www.openarchives.org/OAI/2.0/"; |
|
| 30 |
private static final String DRI_NAMESPACE_URI = "http://www.driver-repository.eu/namespace/dri"; |
|
| 31 |
private static final String RDF_NAMESPACE_URI = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"; |
|
| 32 |
|
|
| 33 |
private SaxonHelper saxonHelper; |
|
| 34 |
private XPathSelector xpathSelectorObjIdentifier; |
|
| 35 |
private XPathSelector xpathSelectorCollectionDate; |
|
| 36 |
private XPathSelector xpathSelectorTransformationDate; |
|
| 37 |
private XPathSelector xpathSelectorDatasourceName; |
|
| 38 |
private XPathSelector xpathSelectorRDF; |
|
| 39 |
|
|
| 40 |
private String connectionString; |
|
| 41 |
private String username; |
|
| 42 |
private String password; |
|
| 43 |
private String defaultBaseURI; |
|
| 44 |
|
|
| 45 |
protected VirtuosoClient(final String connectionString, final String username, final String password, final SaxonHelper saxonHelper, final String defaultBaseURI) |
|
| 46 |
throws SaxonApiException {
|
|
| 47 |
this.connectionString = connectionString; |
|
| 48 |
this.username = username; |
|
| 49 |
this.password = password; |
|
| 50 |
this.saxonHelper = saxonHelper; |
|
| 51 |
this.defaultBaseURI = defaultBaseURI; |
|
| 52 |
prepareXpathSelectors(); |
|
| 53 |
} |
|
| 54 |
|
|
| 55 |
public long feed(final String record){
|
|
| 56 |
if(StringUtils.isBlank(record)){
|
|
| 57 |
log.warn("Got empty record");
|
|
| 58 |
return 0; |
|
| 59 |
} |
|
| 60 |
String objIdentifier = extractFromRecord(record, xpathSelectorObjIdentifier); |
|
| 61 |
if(StringUtils.isBlank(objIdentifier)){
|
|
| 62 |
log.warn("Got record with no objIdentifier -- skipping");
|
|
| 63 |
return 0; |
|
| 64 |
} |
|
| 65 |
String rdfBlock = extractFromRecord(record, xpathSelectorRDF); |
|
| 66 |
if(StringUtils.isBlank(rdfBlock)){
|
|
| 67 |
log.warn("Missing rdf:RDF in record with objIdentifier "+objIdentifier+ " all triples in that named graph will be deleted");
|
|
| 68 |
} |
|
| 69 |
String collectionDate = extractFromRecord(record, xpathSelectorCollectionDate); |
|
| 70 |
String transformationDate = extractFromRecord(record, xpathSelectorTransformationDate); |
|
| 71 |
String datasource = extractFromRecord(record, xpathSelectorDatasourceName); |
|
| 72 |
|
|
| 73 |
Model md = VirtModel.openDatabaseModel(objIdentifier, getConnectionString(), getUsername(), getPassword()); |
|
| 74 |
log.debug("Opened virtuoso model for graph "+objIdentifier);
|
|
| 75 |
md.removeAll(); |
|
| 76 |
log.debug("Removed all triples from graph "+objIdentifier);
|
|
| 77 |
md.read(IOUtils.toInputStream(rdfBlock), getDefaultBaseURI()); |
|
| 78 |
long size = md.size(); |
|
| 79 |
log.info("Graph "+objIdentifier+" now has "+size+" triples");
|
|
| 80 |
|
|
| 81 |
long ntriples = feedProvenance(objIdentifier, collectionDate, transformationDate, datasource); |
|
| 82 |
log.debug("provenance graph updated with "+ntriples+" triples");
|
|
| 83 |
|
|
| 84 |
return size; |
|
| 85 |
} |
|
| 86 |
|
|
| 87 |
long feedProvenance(final String objIdentifier, final String collectionDate, final String transformationDate, final String datasource) {
|
|
| 88 |
Model md = VirtModel.openDatabaseModel("provenance", getConnectionString(), getUsername(), getPassword());
|
|
| 89 |
|
|
| 90 |
//TODO: use prov-o instead: https://www.w3.org/TR/prov-o/#description |
|
| 91 |
Resource r = ResourceFactory.createResource(getRecordDefaultURI(objIdentifier, datasource)); |
|
| 92 |
Statement stmCollFrom = ResourceFactory.createStatement(r, ResourceFactory.createProperty("dnet", "collectedFrom"), ResourceFactory.createPlainLiteral(datasource) );
|
|
| 93 |
//TODO: how to set the type of dates correctly? |
|
| 94 |
Statement stmCollDate = ResourceFactory.createStatement(r, ResourceFactory.createProperty("dnet", "collectedInDate"), ResourceFactory.createPlainLiteral(collectionDate) );
|
|
| 95 |
Statement stmTransDate = ResourceFactory.createStatement(r, ResourceFactory.createProperty("dnet", "transformedInDate"), ResourceFactory.createPlainLiteral(transformationDate) );
|
|
| 96 |
|
|
| 97 |
//let's remove previous provenance statements for this resource: |
|
| 98 |
md.removeAll(r, null, null); |
|
| 99 |
//and add the new ones |
|
| 100 |
md.add(stmCollDate).add(stmCollFrom).add(stmTransDate); |
|
| 101 |
return 3; |
|
| 102 |
} |
|
| 103 |
|
|
| 104 |
public long feed(final Iterable<String> records){
|
|
| 105 |
//TODO: can we do it in parallel? |
|
| 106 |
long count = 0; |
|
| 107 |
for(String r : records) count+=this.feed(r); |
|
| 108 |
return count; |
|
| 109 |
} |
|
| 110 |
|
|
| 111 |
private String getRecordDefaultURI(final String objIdentifier, final String datasource){
|
|
| 112 |
return defaultBaseURI+datasource+"/"+objIdentifier; |
|
| 113 |
} |
|
| 114 |
|
|
| 115 |
private void prepareXpathSelectors() throws SaxonApiException {
|
|
| 116 |
Map<String, String> namespaces = Maps.newHashMap(); |
|
| 117 |
namespaces.put("oai", OAI_NAMESPACE_URI);
|
|
| 118 |
namespaces.put("dri", DRI_NAMESPACE_URI);
|
|
| 119 |
namespaces.put("rdf", RDF_NAMESPACE_URI);
|
|
| 120 |
xpathSelectorObjIdentifier = this.saxonHelper.help().prepareXPathSelector("//oai:header/dri:objIdentifier/text()", namespaces);
|
|
| 121 |
xpathSelectorCollectionDate = this.saxonHelper.help().prepareXPathSelector("//oai:header/dri:dateOfCollection/text()", namespaces);
|
|
| 122 |
xpathSelectorTransformationDate = this.saxonHelper.help().prepareXPathSelector("//oai:header/dri:dateOfTransformation/text()", namespaces);
|
|
| 123 |
xpathSelectorDatasourceName = this.saxonHelper.help().prepareXPathSelector("//oai:header/dri:datasourcename/text()", namespaces);
|
|
| 124 |
xpathSelectorRDF = this.saxonHelper.help().prepareXPathSelector("//oai:metadata/rdf:RDF", namespaces);
|
|
| 125 |
} |
|
| 126 |
|
|
| 127 |
private String extractFromRecord(final String record, final XPathSelector xPathSelector) {
|
|
| 128 |
try {
|
|
| 129 |
return this.saxonHelper.help().setSerializerProperty(Property.OMIT_XML_DECLARATION, "yes").evaluateSingleAsString(record, xPathSelector); |
|
| 130 |
} catch (SaxonApiException e) {
|
|
| 131 |
throw new RuntimeException("Cannot extract content ", e);
|
|
| 132 |
} |
|
| 133 |
} |
|
| 134 |
|
|
| 135 |
public String getConnectionString() {
|
|
| 136 |
return connectionString; |
|
| 137 |
} |
|
| 138 |
|
|
| 139 |
public String getUsername() {
|
|
| 140 |
return username; |
|
| 141 |
} |
|
| 142 |
|
|
| 143 |
public String getPassword() {
|
|
| 144 |
return password; |
|
| 145 |
} |
|
| 146 |
|
|
| 147 |
public SaxonHelper getSaxonHelper() {
|
|
| 148 |
return saxonHelper; |
|
| 149 |
} |
|
| 150 |
|
|
| 151 |
public String getDefaultBaseURI() {
|
|
| 152 |
return defaultBaseURI; |
|
| 153 |
} |
|
| 154 |
} |
|
| modules/dnet-parthenos-publisher/trunk/src/main/resources/eu/dnetlib/parthenos/virtuoso/applicationContext-virtuoso.xml | ||
|---|---|---|
| 1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
| 2 |
<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
|
| 3 |
xmlns:p="http://www.springframework.org/schema/p" |
|
| 4 |
xmlns="http://www.springframework.org/schema/beans" |
|
| 5 |
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd"> |
|
| 6 |
|
|
| 7 |
<bean id="virtuosoClientFactory" class="eu.dnetlib.parthenos.virtuoso.VirtuosoClientFactory" |
|
| 8 |
p:connectionString="${virtuoso.connectionstring}" p:password="${virtuoso.pwd}" p:username="${virtuoso.usr}"
|
|
| 9 |
p:defaultBaseURI="${virtuoso.uri.base.default"/>
|
|
| 10 |
|
|
| 11 |
</beans> |
|
| modules/dnet-parthenos-publisher/trunk/src/main/resources/eu/dnetlib/parthenos/virtuoso/applicationContext-virtuoso.properties | ||
|---|---|---|
| 1 |
virtuoso.connectionstring = jdbc:virtuoso://localhost:1111 |
|
| 2 |
virtuoso.pwd = dba |
|
| 3 |
virtuoso.usr = dba |
|
| 4 |
virtuoso.uri.base.default = http://parthenos.d4science.org/handle/ |
|
| modules/dnet-parthenos-publisher/trunk/pom.xml | ||
|---|---|---|
| 17 | 17 |
</scm> |
| 18 | 18 |
<dependencies> |
| 19 | 19 |
<dependency> |
| 20 |
<groupId>eu.dnetlib</groupId> |
|
| 21 |
<artifactId>dnet-core-components</artifactId> |
|
| 22 |
<version>[2.0.0-SAXONHE-SNAPSHOT]</version> |
|
| 23 |
</dependency> |
|
| 24 |
<dependency> |
|
| 20 | 25 |
<groupId>virtuoso</groupId> |
| 21 | 26 |
<artifactId>jena-driver</artifactId> |
| 22 | 27 |
<version>3.0</version> |
Also available in: Unified diff
VirtuosoClient and some tests assuming there is a local virtuoso server (hence ignored)