D-Net

package eu.dnetlib.parthenos.virtuoso;

import java.util.Map;

import eu.dnetlib.parthenos.publisher.ParthenosPublisherException;

import eu.dnetlib.parthenos.publisher.SaxonHelper;

import net.sf.saxon.s9api.SaxonApiException;

import net.sf.saxon.s9api.Serializer;

import net.sf.saxon.s9api.XPathSelector;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.jena.datatypes.xsd.XSDDatatype;

import org.apache.jena.ext.com.google.common.collect.Maps;

import org.apache.jena.rdf.model.*;

import virtuoso.jena.driver.VirtModel;

/**

 * Created by Alessia Bardi on 12/07/2017.

 *

 * @author Alessia Bardi

 */

public class VirtuosoClient {

	private static final Log log = LogFactory.getLog(VirtuosoClient.class);

	private static final String OAI_NAMESPACE_URI = "http://www.openarchives.org/OAI/2.0/";

	private static final String DRI_NAMESPACE_URI = "http://www.driver-repository.eu/namespace/dri";

	private static final String RDF_NAMESPACE_URI = "http://www.w3.org/1999/02/22-rdf-syntax-ns#";

	private static final String PROVENANCE_GRAPH = "provenance";

	public static Property IS_API_OF = ResourceFactory.createProperty("dnet", "isApiOf");

	public static Property COLL_FROM = ResourceFactory.createProperty("dnet", "collectedFrom");

	public static Property COLL_IN_DATE = ResourceFactory.createProperty("dnet", "collectedInDate");

	public static Property TRANS_IN_DATE = ResourceFactory.createProperty("dnet", "transformedInDate");

	private SaxonHelper saxonHelper;

	private XPathSelector xpathSelectorObjIdentifier;

	private XPathSelector xpathSelectorCollectionDate;

	private XPathSelector xpathSelectorTransformationDate;

	private XPathSelector xpathSelectorDatasourceName;

	private XPathSelector xpathSelectorDatasourceApi;

	private XPathSelector xpathSelectorRDF;

	private String connectionString;

	private String username;

	private String password;

	private String defaultBaseURI;

	protected VirtuosoClient(final String connectionString,

			final String username,

			final String password,

			final SaxonHelper saxonHelper,

			final String defaultBaseURI)

			throws ParthenosPublisherException {

		this.connectionString = connectionString;

		this.username = username;

		this.password = password;

		this.saxonHelper = saxonHelper;

		this.defaultBaseURI = defaultBaseURI;

		try {

			prepareXpathSelectors();

		}catch(SaxonApiException e){

			log.error(e.getMessage());

			throw new ParthenosPublisherException(e);

		}

	}

	//TODO: exploit new method eu.dnetlib.parthenos.publisher.SaxonHelper.Helper.parseXML() to avoid re-parsing the full record.

	public long feed(final String record) throws ParthenosPublisherException{

		Model md = null ;

		try {

			if (StringUtils.isBlank(record)) {

				log.warn("Got empty record");

				return 0;

			}

			String objIdentifier = extractFromRecord(record, xpathSelectorObjIdentifier);

			if (StringUtils.isBlank(objIdentifier)) {

				log.warn("Got record with no objIdentifier -- skipping");

				return 0;

			}

			String rdfBlock = extractFromRecord(record, xpathSelectorRDF);

			if (StringUtils.isBlank(rdfBlock)) {

				log.warn("Missing rdf:RDF in record with objIdentifier " + objIdentifier + " all triples in that named graph will be deleted");

			}

			String collectionDate = extractFromRecord(record, xpathSelectorCollectionDate);

			String transformationDate = extractFromRecord(record, xpathSelectorTransformationDate);

			String datasource = extractFromRecord(record, xpathSelectorDatasourceName);

			String dsInterface = extractFromRecord(record, xpathSelectorDatasourceApi);

			String namedGraph = getRecordDefaultURI(objIdentifier, dsInterface);

			log.debug("Trying to open the database model " + namedGraph+", connection string "+getConnectionString());

			md = VirtModel.openDatabaseModel(namedGraph, getConnectionString(), getUsername(), getPassword());

			log.debug("Opened virtuoso model for graph " + namedGraph);

			md.removeAll();

			log.debug("Removed all triples from graph " + namedGraph);

			md.read(IOUtils.toInputStream(rdfBlock, "UTF-8"), getDefaultBaseURI());

			long size = md.size();

			log.debug("Graph " + namedGraph + " now has " + size + " triples");

			long ntriples = feedProvenance(namedGraph, collectionDate, transformationDate, datasource, dsInterface);

			log.debug("provenance graph for " + namedGraph + " updated with " + ntriples + " triples");

			md.close();

			return size;

		}catch(Throwable e){

			if (md != null && !md.isClosed()) md.close();

			log.error(e);

			throw new ParthenosPublisherException(e);

		}

	}

	long feedProvenance(final String namedGraphURI, final String collectionDate, final String transformationDate, final String datasource, final String api) {

		Model md = VirtModel.openDatabaseModel(PROVENANCE_GRAPH, getConnectionString(), getUsername(), getPassword());

		//TODO: use prov-o instead: https://www.w3.org/TR/prov-o/#description

		Resource rApi = ResourceFactory.createResource(defaultBaseURI + api);

		Resource r = ResourceFactory.createResource(namedGraphURI);

		Statement stmApi =

				ResourceFactory.createStatement(rApi, IS_API_OF, ResourceFactory.createPlainLiteral(datasource));

		Statement stmCollFrom =

				ResourceFactory.createStatement(r, COLL_FROM, rApi);

		Statement stmCollDate = ResourceFactory

				.createStatement(r, COLL_IN_DATE, ResourceFactory.createTypedLiteral(collectionDate, XSDDatatype.XSDdateTime));

		Statement stmTransDate = ResourceFactory

				.createStatement(r, TRANS_IN_DATE, ResourceFactory.createTypedLiteral(transformationDate, XSDDatatype.XSDdateTime));

		//let's remove previous provenance statements for this resource:

		md.removeAll(r, null, null);

		//and add the new ones

		md.add(stmApi).add(stmCollFrom).add(stmCollDate).add(stmTransDate);

		md.close();

		return 3;

	}

	public long feed(final Iterable<String> records) throws ParthenosPublisherException {

		//TODO: can we do it in parallel? if all records have different objIdentifier it is safe, and this must be the case anyway, because the source of records is a D-Net mdstore.

		long count = 0;

		for (String r : records) count += this.feed(r);

		return count;

	}

	/**

	 * Delete all triples in named graphs collected from the given api

	 * @param api the id of the API

	 * @return the number of triples deleted from the named graphs associated to the given api

	 */

	public long drop(final String api){

		Model prov = VirtModel.openDatabaseModel(PROVENANCE_GRAPH, getConnectionString(), getUsername(), getPassword());

		//look for all named graphs associated to the api

		Resource rApi = ResourceFactory.createResource(defaultBaseURI + api);

		long deletedTriples = 0;

		final ResIterator resIterator = prov.listSubjectsWithProperty(COLL_FROM, rApi);

		while (resIterator.hasNext()) {

			Resource namedGraphURI = resIterator.nextResource();

			//delete all triples belonging to the r named graph

			deletedTriples += dropNamedGraph(namedGraphURI.getURI());

			//delete the named graph from the provenance graph

			prov.removeAll(namedGraphURI, null, null);

		}

		//delete the api from the provenance graph

		prov.removeAll(null, null, rApi);

		prov.removeAll(rApi, null, null);

		prov.close();

		return deletedTriples;

	}

	private long dropNamedGraph(String namedGraphURI){

		Model namedGraph = VirtModel.openDatabaseModel(namedGraphURI, getConnectionString(), getUsername(), getPassword());

		long deletedTriples = namedGraph.size();

		namedGraph.removeAll();

		namedGraph.close();

		return deletedTriples;

	}

	private String getRecordDefaultURI(final String objIdentifier, final String datasourceApi) {

		return defaultBaseURI + datasourceApi + "/" + objIdentifier;

	}

	private void prepareXpathSelectors() throws SaxonApiException {

		Map<String, String> namespaces = Maps.newHashMap();

		namespaces.put("oai", OAI_NAMESPACE_URI);

		namespaces.put("dri", DRI_NAMESPACE_URI);

		namespaces.put("rdf", RDF_NAMESPACE_URI);

		xpathSelectorObjIdentifier = this.saxonHelper.help().prepareXPathSelector("//oai:header/dri:objIdentifier/text()", namespaces);

		xpathSelectorCollectionDate = this.saxonHelper.help().prepareXPathSelector("//oai:header/dri:dateOfCollection/text()", namespaces);

		xpathSelectorTransformationDate = this.saxonHelper.help().prepareXPathSelector("//oai:header/dri:dateOfTransformation/text()", namespaces);

		xpathSelectorDatasourceName = this.saxonHelper.help().prepareXPathSelector("//oai:header/dri:datasourcename/text()", namespaces);

		xpathSelectorDatasourceApi = this.saxonHelper.help().prepareXPathSelector("//oai:header/dri:datasourceapi/text()", namespaces);

		xpathSelectorRDF = this.saxonHelper.help().prepareXPathSelector("//oai:metadata/rdf:RDF", namespaces);

	}

	private String  extractFromRecord(final String record, final XPathSelector xPathSelector) {

		try {

			return this.saxonHelper.help().setSerializerProperty(Serializer.Property.OMIT_XML_DECLARATION, "yes").evaluateSingleAsString(record, xPathSelector);

		} catch (SaxonApiException e) {

			log.error(e);

			throw new RuntimeException("Cannot extract content ", e);

		}

	}

	public String getConnectionString() {

		return connectionString;

	}

	public String getUsername() {

		return username;

	}

	public String getPassword() {

		return password;

	}

	public SaxonHelper getSaxonHelper() {

		return saxonHelper;

	}

	public String getDefaultBaseURI() {

		return defaultBaseURI;

	}

}