D-Net

import eu.dnetlib.parthenos.publisher.ParthenosPublisherException;

import eu.dnetlib.parthenos.publisher.SaxonHelper;

	public void prepare() throws ParthenosPublisherException {

	public void testRecord() throws ParthenosPublisherException {

package eu.dnetlib.parthenos.publisher;

import java.io.IOException;

import eu.dnetlib.parthenos.ParthenosPublisherApplication;

import org.apache.commons.io.IOUtils;

import org.apache.jena.graph.Triple;

import org.apache.jena.util.iterator.ExtendedIterator;

import org.junit.Ignore;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;

import org.springframework.boot.test.context.SpringBootTest;

import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;

import org.springframework.core.io.ClassPathResource;

import org.springframework.test.context.TestPropertySource;

import org.springframework.test.context.junit4.SpringRunner;

import org.springframework.test.web.servlet.MockMvc;

import virtuoso.jena.driver.VirtGraph;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertNotEquals;

import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;

import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;

/**

 * Created by Alessia Bardi on 29/08/2017.

 * Integration test.

 *

 * @author Alessia Bardi

 */

@RunWith(SpringRunner.class)

@SpringBootTest(

		webEnvironment = WebEnvironment.RANDOM_PORT,

		classes = ParthenosPublisherApplication.class)

@AutoConfigureMockMvc

@TestPropertySource(locations = "classpath:application-integrationtest.properties")

//Ignore test because ot requires a running virtuoso server

@Ignore

public class ParthenosPublisherTest {

	private String recordPath = "eu/dnetlib/parthenos/virtuoso/test_record_plain.xml";

	@Value("${virtuoso.connectionstring}")

	private String connectionString;

	@Value("${virtuoso.usr}")

	private String testUser ;

	@Value("${virtuoso.pwd}")

	private String testPwd ;

	@Value("${virtuoso.uri.base.default}")

	private String defaultURIBaseURl;

	@Autowired

	private MockMvc mvc;

	@Test

	public void publishOK() throws Exception {

		String graphName = defaultURIBaseURl + "api_________::parthenos___::parthenos::topLevel/parthenos___::8d777f385d3dfec8815d20f7496026dc";

		VirtGraph graph = new VirtGraph(graphName, connectionString, testUser, testPwd);

		//ensure we have an empty graph

		graph.clear();

		assertEquals(0, graph.getCount());

		mvc.perform(post("/publish").param("record",  getString(recordPath)).param("serializationMode", "plain"))

				.andExpect(status().isOk());

		int count = graph.getCount();

		System.out.println(graphName +"has "+count+" triples:");

		assertNotEquals(0, graph.getCount());

		final ExtendedIterator<Triple> triples = graph.find();

		while(triples.hasNext()){

			System.out.println(triples.next());

		}

		//cleanup the virtuoso graph

		graph.clear();

		assertEquals(0, graph.getCount());

	}

	private String getString(final String classpath) {

		try {

			final ClassPathResource resource = new ClassPathResource(classpath);

			return IOUtils.toString(resource.getInputStream(), "UTF-8");

		}catch(IOException e){

			return null;

		}

	}

}

package eu.dnetlib.parthenos.publisher;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest;

import org.springframework.boot.test.mock.mockito.MockBean;

import org.springframework.test.context.junit4.SpringRunner;

import org.springframework.test.web.servlet.MockMvc;

import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;

import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;

/**

 * Created by Alessia Bardi on 29/08/2017.

 *

 * @author Alessia Bardi

 */

@RunWith(SpringRunner.class)

@WebMvcTest(ParthenosPublisherController.class)

public class ParthenosPublisherControllerTest {

	@Autowired

	private MockMvc mvc;

	@MockBean

	private ParthenosPublisherHelper parthenosPublisherHelper;

	@Test

	public void publishOK() throws Exception {

		mvc.perform(post("/publish").param("record", "<aTestRecord>value</aTestRecord>").param("parthenosTarget", "VIRTUOSO"))

				.andExpect(status().isOk());

	}

}

virtuoso.connectionstring = jdbc:virtuoso://localhost:1111

virtuoso.pwd = dba

virtuoso.usr = dba

virtuoso.uri.base.default =  http://parthenos.test/handle/

    <orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.4" level="project" />

    <orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />

    <orderEntry type="library" scope="TEST" name="Maven: org.ow2.asm:asm:5.0.3" level="project" />

    <orderEntry type="library" scope="TEST" name="Maven: org.springframework:spring-test:4.3.10.RELEASE" level="project" />

    <orderEntry type="library" name="Maven: org.springframework:spring-context:4.3.10.RELEASE" level="project" />

    <orderEntry type="library" name="Maven: org.springframework:spring-aop:4.3.10.RELEASE" level="project" />

    <orderEntry type="library" name="Maven: org.springframework:spring-beans:4.3.10.RELEASE" level="project" />

    <orderEntry type="library" name="Maven: org.springframework:spring-expression:4.3.10.RELEASE" level="project" />

    <orderEntry type="library" name="Maven: org.springframework:spring-webmvc:4.3.10.RELEASE" level="project" />

    <orderEntry type="library" name="Maven: com.google.guava:guava:21.0" level="project" />

    <orderEntry type="library" name="Maven: org.apache.solr:solr-solrj:5.5.4" level="project" />

    <orderEntry type="library" name="Maven: commons-io:commons-io:2.4" level="project" />

    <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.5.3" level="project" />

    <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.4.6" level="project" />

    <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpmime:4.5.3" level="project" />

    <orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.4.6" level="project" />

    <orderEntry type="library" name="Maven: org.codehaus.woodstox:stax2-api:3.1.4" level="project" />

    <orderEntry type="library" name="Maven: org.codehaus.woodstox:woodstox-core-asl:4.4.1" level="project" />

    <orderEntry type="library" name="Maven: org.noggit:noggit:0.6" level="project" />

    <orderEntry type="library" name="Maven: org.slf4j:jcl-over-slf4j:1.7.25" level="project" />

package eu.dnetlib.parthenos;

/**

 * Created by Alessia Bardi on 09/08/2017.

 *

 * @author Alessia Bardi

 */

import org.springframework.boot.SpringApplication;

import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication

public class ParthenosPublisherApplication {

	public static void main(String[] args) {

		SpringApplication.run(ParthenosPublisherApplication.class, args);

	}

}

import eu.dnetlib.parthenos.publisher.ParthenosPublisherException;

import eu.dnetlib.parthenos.publisher.SaxonHelper;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.stereotype.Component;

@Component

	@Value("${virtuoso.connectionstring}")

	@Value("${virtuoso.usr}")

	@Value("${virtuoso.pwd}")

	@Value("${virtuoso.uri.base.default}")

	public VirtuosoClient getVirtuosoClient() throws ParthenosPublisherException {

import java.io.IOException;

import eu.dnetlib.parthenos.publisher.ParthenosPublisherException;

import eu.dnetlib.parthenos.publisher.SaxonHelper;

import org.apache.jena.ext.com.google.common.collect.Maps;

			throws ParthenosPublisherException {

		try {

			prepareXpathSelectors();

		}catch(SaxonApiException e){

			throw new ParthenosPublisherException(e);

		}

	public long feed(final String record) throws ParthenosPublisherException{

		try {

			if (StringUtils.isBlank(record)) {

				log.warn("Got empty record");

				return 0;

			}

			String objIdentifier = extractFromRecord(record, xpathSelectorObjIdentifier);

			if (StringUtils.isBlank(objIdentifier)) {

				log.warn("Got record with no objIdentifier -- skipping");

				return 0;

			}

			String rdfBlock = extractFromRecord(record, xpathSelectorRDF);

			if (StringUtils.isBlank(rdfBlock)) {

				log.warn("Missing rdf:RDF in record with objIdentifier " + objIdentifier + " all triples in that named graph will be deleted");

			}

			String collectionDate = extractFromRecord(record, xpathSelectorCollectionDate);

			String transformationDate = extractFromRecord(record, xpathSelectorTransformationDate);

			String datasource = extractFromRecord(record, xpathSelectorDatasourceName);

			String dsInterface = extractFromRecord(record, xpathSelectorDatasourceApi);

			String namedGraph = getRecordDefaultURI(objIdentifier, dsInterface);

			Model md = VirtModel.openDatabaseModel(namedGraph, getConnectionString(), getUsername(), getPassword());

			log.debug("Opened virtuoso model for graph " + namedGraph);

			md.removeAll();

			log.debug("Removed all triples from graph " + namedGraph);

			md.read(IOUtils.toInputStream(rdfBlock, "UTF-8"), getDefaultBaseURI());

			long size = md.size();

			log.info("Graph " + namedGraph + " now has " + size + " triples");

			long ntriples = feedProvenance(namedGraph, collectionDate, transformationDate, datasource, dsInterface);

			log.debug("provenance graph for " + namedGraph + " updated with " + ntriples + " triples");

			return size;

		}catch(IOException e){

			throw new ParthenosPublisherException(e);

		}

	public long feed(final Iterable<String> records) throws ParthenosPublisherException {

package eu.dnetlib.parthenos.publisher;

/**

 * Created by Alessia Bardi on 29/08/2017.

 *

 * @author Alessia Bardi

 */

public class ParthenosPublisherException extends Exception{

	public ParthenosPublisherException() {

	}

	public ParthenosPublisherException(final String message) {

		super(message);

	}

	public ParthenosPublisherException(final String message, final Throwable cause) {

		super(message, cause);

	}

	public ParthenosPublisherException(final Throwable cause) {

		super(cause);

	}

	public ParthenosPublisherException(final String message, final Throwable cause, final boolean enableSuppression, final boolean writableStackTrace) {

		super(message, cause, enableSuppression, writableStackTrace);

	}

}

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.stereotype.Component;

@Component

	private static final Log log = LogFactory.getLog(ParthenosPublisherHelper.class);

	public enum ParthenosTargets{

		ALL, SOLR, VIRTUOSO, JRR, OAI

	}

	public void publish(final String record, final ParthenosTargets target) throws ParthenosPublisherException {

		switch(target){

		case ALL:

			publishVirtuoso(record);

			break;

		case VIRTUOSO:

			publishVirtuoso(record);

			break;

			default: throw new ParthenosPublisherException("Target "+target+" not supported yet");

		}

	}

	public long unpublish(final String datasourceInterface, final ParthenosTargets target) throws ParthenosPublisherException {

		long res = 0;

		switch(target){

		case ALL:

			res = unpublishVirtuoso(datasourceInterface);

			break;

		case VIRTUOSO:

			res = unpublishVirtuoso(datasourceInterface);

			break;

		default: throw new ParthenosPublisherException("Target "+target+" not supported yet");

		}

		return res;

	}

	private void publishVirtuoso(final String record) throws ParthenosPublisherException {

		log.debug("Publishing on virtuoso");

		virtuosoClient.feed(record);

	private long unpublishVirtuoso(final String datasourceInterface) throws ParthenosPublisherException {

		log.debug("Unublishing from virtuoso "+datasourceInterface);

		VirtuosoClient virtuosoClient = this.virtuosoClientFactory.getVirtuosoClient();

		return virtuosoClient.drop(datasourceInterface);

	}

//	private void publishSolr(final String record){

//		SolrClient solr = new CloudSolrClient.Builder().withSolrUrl("http://localhost:8983/solr").build();

//

//	}

import eu.dnetlib.parthenos.publisher.ParthenosPublisherHelper.ParthenosTargets;

import org.apache.commons.lang3.StringUtils;

import org.springframework.beans.factory.annotation.Autowired;

	//TODO: to nicely handle arrors, follow https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#boot-features-error-handling

	@Autowired

	private ParthenosPublisherHelper parthenosPublisherHelper;

	@RequestMapping(value = "/publish", method = RequestMethod.POST)

	public void publish(@RequestParam final String record, @RequestParam(required = false) String parthenosTarget) throws ParthenosPublisherException {

		this.parthenosPublisherHelper.publish(record, getTarget(parthenosTarget));

	@RequestMapping(value = "/unpublish", method = RequestMethod.GET)

	public long unpublish(@RequestParam final String datasourceApi, @RequestParam(required = false) String parthenosTarget) throws ParthenosPublisherException {

		return this.parthenosPublisherHelper.unpublish(datasourceApi, getTarget(parthenosTarget));

	}

	private ParthenosTargets getTarget(String value){

		ParthenosTargets target = ParthenosTargets.ALL;

		if (StringUtils.isNotBlank(value)) {

			target = ParthenosTargets.valueOf(value);

		}

		return target;

package eu.dnetlib.parthenos.publisher;

import java.io.StringReader;

import java.util.Map;

import java.util.Map.Entry;

import javax.xml.transform.sax.SAXSource;

import com.google.common.collect.Maps;

import net.sf.saxon.s9api.*;

import net.sf.saxon.s9api.Serializer.Property;

import org.springframework.stereotype.Component;

import org.xml.sax.InputSource;

/**

 * Created by alessia on 10/03/17.

 */

@Component

public class SaxonHelper {

	private Processor xmlProcessor = new Processor(false);

	public Helper help(){

		return new Helper();

	}

	public class Helper{

		private Serializer serializer;

		Helper() {

			this.serializer = xmlProcessor.newSerializer();

			serializer.setOutputProperty(Property.METHOD, "xml");

			serializer.setOutputProperty(Property.INDENT, "yes");

		}

		public Helper setSerializerProperty(Property p, String value){

			serializer.setOutputProperty(p, value);

			return this;

		}

		/**

		 * Evaluate the given xpath on the given XML source.

		 *

		 * @param xmlSource  XML source string

		 * @param xpath      the xpath to evaluate

		 * @param namespaces the map of namespaces to be declared to evaluate the xpath

		 * @return an XdmItem resulting from the evaluation of the xpath. Returns null if nothing matches.

		 */

		public XdmItem evaluateSingle(final String xmlSource, final String xpath, final Map<String, String> namespaces) throws SaxonApiException {

			XPathSelector xpathSelector = prepareXPathSelector(xpath, namespaces);

			return evaluateSingle(xmlSource, xpathSelector);

		}

		/**

		 * Applies the given xpath selector on the given XML source.

		 *

		 * @param xmlSource  XML source string

		 * @param xpathSelector    the configured xpath  selector to apply

		 * @return an XdmItem resulting from the evaluation of the xpath. Returns null if nothing matches.

		 */

		public XdmItem evaluateSingle(final String xmlSource, final XPathSelector xpathSelector) throws SaxonApiException {

			SAXSource source = new SAXSource(new InputSource(new StringReader(xmlSource)));

			DocumentBuilder docBuilder = xmlProcessor.newDocumentBuilder();

			XdmNode xdmNode = docBuilder.build(source);

			xpathSelector.setContextItem(xdmNode);

			return xpathSelector.evaluateSingle();

		}

		/**

		 * Get an XPathSelector for the given xpath and namespaces.

		 * @param xpath the xpath

		 * @param namespaces the map of namespaces to be declared to evaluate the xpath

		 * @return XPathSelector

		 * @throws SaxonApiException

		 */

		public XPathSelector prepareXPathSelector(final String xpath,final Map<String, String> namespaces) throws SaxonApiException {

			XPathCompiler compiler = xmlProcessor.newXPathCompiler();

			for (Entry<String, String> ns : namespaces.entrySet()) {

				compiler.declareNamespace(ns.getKey(), ns.getValue());

			}

			XPathExecutable xpathExecutable = compiler.compile(xpath);

			XPathSelector xpathSelector = xpathExecutable.load();

			return xpathSelector;

		}

		/**

		 * Get an XPathSelector for the given xpath and namespaces.

		 * @param xpath the xpath

		 * @param nsPrefix namespace prefix

		 * @param ns namespace URI

		 * @return XPathSelector

		 * @throws SaxonApiException

		 */

		public XPathSelector prepareXPathSelector(final String xpath,final String nsPrefix, final String ns) throws SaxonApiException {

			Map<String, String> map = Maps.newHashMap();

			map.put(nsPrefix, ns);

			return prepareXPathSelector(xpath, map);

		}

		/**

		 *  Evaluate the given xpath on the given XML source.

		 * @param xmlSource  XML source string

		 * @param xpath      the xpath to evaluate

		 * @param nsPrefix namespace prefix

		 * @param ns namespace URI

		 * @return an XdmItem resulting from the evaluation of the xpath. Returns null if nothing matches.

		 * @throws SaxonApiException

		 *

		 */

		public XdmItem evaluateSingle(final String xmlSource, final String xpath, final String nsPrefix, final String ns) throws SaxonApiException {

			Map<String, String> map = Maps.newHashMap();

			map.put(nsPrefix, ns);

			return evaluateSingle(xmlSource, xpath, map);

		}

		/**

		 * Evaluate the given xpath on the given XML source.

		 *

		 * @param xmlSource  XML source string

		 * @param xpath      the xpath to evaluate. xpath must evaluate to an xdmNode

		 * @param namespaces the map of namespaces to be declared to evaluate the xpath

		 * @return the XML serialization as string of the XdmItem resulting from the evaluation of the xpath. Returns null if nothing matches.

		 * @throws ClassCastException if xpath is not evaluated to a node.

		 */

		public String evaluateSingleAsString(final String xmlSource, final String xpath, final Map<String, String> namespaces) throws SaxonApiException {

			XdmItem item = evaluateSingle(xmlSource, xpath, namespaces);

			if(item != null){

				XdmNode node = (XdmNode) item;

				return serializer.serializeNodeToString(node);

			}

			else return null;

		}

		/**

		 * Applies the given xpath selector on the given XML source.

		 *

		 * @param xmlSource  XML source string

		 * @param xpathSelector    the configured xpath  selector to apply

		 * @return the XML serialization as string of the XdmItem resulting from the evaluation of the xpath. Returns null if nothing matches.

		 * @throws ClassCastException if xpath is not evaluated to a node.

		 */

		public String evaluateSingleAsString(final String xmlSource, final XPathSelector xpathSelector) throws SaxonApiException {

			XdmItem item = evaluateSingle(xmlSource, xpathSelector);

			if(item != null){

				XdmNode node = (XdmNode) item;

				return serializer.serializeNodeToString(node);

			}

			else return null;

		}

		/**

		 * Evaluate the given xpath on the given XML source.

		 *

		 * @param xmlSource  XML source string

		 * @param xpath      the xpath to evaluate. xpath must evaluate to an xdmNode

		 * @param nsPrefix namespace prefix

		 * @param ns namespace URI

		 * @return the XML serialization as string of the XdmItem resulting from the evaluation of the xpath. Returns null if nothing matches.

		 * @throws ClassCastException if xpath is not evaluated to a node.

		 */

		public String evaluateSingleAsString(final String xmlSource, final String xpath, final String nsPrefix, final String ns) throws SaxonApiException {

			Map<String, String> map = Maps.newHashMap();

			map.put(nsPrefix, ns);

			return evaluateSingleAsString(xmlSource, xpath, map);

		}

	}

}

server.contextPath                  =   /parthenos

virtuoso.connectionstring = jdbc:virtuoso://localhost:1111

virtuoso.pwd = dba

virtuoso.usr = dba

virtuoso.uri.base.default =  http://parthenos.d4science.org/handle/

		<!--&lt;!&ndash; Needed for SaxonHelper class &ndash;&gt;-->

		<!--<dependency>-->

			<!--<groupId>eu.dnetlib</groupId>-->

			<!--<artifactId>dnet-core-components</artifactId>-->

			<!--<version>2.0.0-SAXONHE-SNAPSHOT</version>-->

		<!--</dependency>-->

			<groupId>net.sf.saxon</groupId>

			<artifactId>Saxon-HE</artifactId>

			<version>9.5.1-5</version>

		<dependency>

			<groupId>com.google.guava</groupId>

			<artifactId>guava</artifactId>

			<version>21.0</version>

		</dependency>

		<dependency>

			<groupId>org.apache.solr</groupId>

			<artifactId>solr-solrj</artifactId>

			<version>[5.0.0,6.0.0)</version>

		</dependency>