D-Net

package eu.dnetlib.parthenos.registry;

import java.io.IOException;

import java.io.StringWriter;

import java.util.Map;

import java.util.Set;

import java.util.UUID;

import com.fasterxml.jackson.core.JsonFactory;

import com.fasterxml.jackson.core.JsonGenerator;

import com.google.common.collect.Maps;

import com.google.common.collect.Sets;

import eu.dnetlib.parthenos.CRM;

import eu.dnetlib.parthenos.CRMdig;

import eu.dnetlib.parthenos.CRMpe;

import eu.dnetlib.parthenos.publisher.ParthenosPublisherException;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.jena.assembler.AssemblerHelp;

import org.apache.jena.assembler.exceptions.AmbiguousSpecificTypeException;

import org.apache.jena.ontology.OntModel;

import org.apache.jena.ontology.OntModelSpec;

import org.apache.jena.rdf.model.*;

import org.apache.jena.vocabulary.RDF;

import org.gcube.informationsystem.resourceregistry.api.exceptions.ResourceRegistryException;

import org.gcube.informationsystem.resourceregistry.client.ResourceRegistryClient;

import org.gcube.informationsystem.resourceregistry.client.ResourceRegistryClientFactory;

import org.gcube.informationsystem.resourceregistry.publisher.ResourceRegistryPublisher;

import org.gcube.informationsystem.resourceregistry.publisher.ResourceRegistryPublisherFactory;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.stereotype.Component;

/**

 * Created by Alessia Bardi on 26/09/2017.

 *

 * @author Alessia Bardi

 */

@Component

public class GCubeResourceRegistrator {

	private static final Log log = LogFactory.getLog(GCubeResourceRegistrator.class);

	private OntModel baseModel;

	@Autowired

	private FacetWriter facetWriter;

	@Autowired

	private RelWriter relWriter;

	private ResourceRegistryPublisher resourceRegistryPublisher;

	private ResourceRegistryClient resourceRegistryClient;

	public GCubeResourceRegistrator() {

		this.resourceRegistryPublisher = ResourceRegistryPublisherFactory.create();

		this.resourceRegistryClient = ResourceRegistryClientFactory.create();

		baseModel = ModelFactory.createOntologyModel(OntModelSpec.RDFS_MEM_TRANS_INF);

		baseModel.read(CRMpe.RDFS_URL);

		baseModel.read(CRM.RDFS_URL);

		baseModel.read(CRMdig.RDFS_URL);

	}

	public int unregister(final String datasourceApi){

		//TODO: implement me

		return -1;

	}

	/**

	 * Registers resources and relationships into the Parthenos Registry

	 *

	 * @param rdfRecord RDF record in RDF/XML PLAIN format

	 */

	public void register(final String rdfRecord, final String objIdentifier) throws IOException, ResourceRegistryException, ParthenosPublisherException {

		if (StringUtils.isBlank(rdfRecord)) {

			log.warn("Got empty record "+objIdentifier);

		} else {

			InfModel model = loadBaseModel();

			model.read(IOUtils.toInputStream(rdfRecord, "UTF-8"), CRMpe.NS);

			JsonFactory jsonFactory = new JsonFactory();

			Set<String> uriProcessed = Sets.newHashSet();

			/* Maps URI to ParthenosRegistryResource (with uuids) */

			Map<String, ParthenosRegistryResource> idMap = Maps.newHashMap();

			/* Resource */

			int registeredServices = processServices(model, uriProcessed, jsonFactory, idMap);

			int registeredDigitalObjects = processDigitalObjects(model, uriProcessed, jsonFactory, idMap);

			int registeredActors = processActors(model, uriProcessed, jsonFactory, idMap);

			int registeredProjects = processProjects(model, uriProcessed, jsonFactory, idMap);

			int total = registeredActors+registeredDigitalObjects+registeredProjects+registeredServices;

			log.debug(String.format("Registered %d resources from record with objIdentifier %s", total, objIdentifier));

			/* Relationships */

			int relTotal = 0;

			for(String uri : uriProcessed){

				relTotal += processRelationships(model, uri, idMap);

			}

			log.debug(String.format("Registered %d relationships from record with objIdentifier %s", relTotal, objIdentifier));

		}

	}

	private int processRelationships(final InfModel model, final String uri, final Map<String, ParthenosRegistryResource> idMap)

			throws IOException, ResourceRegistryException, ParthenosPublisherException {

		Resource subject = model.getResource(uri);

		String uuid = idMap.get(uri).getUuid();

		JsonFactory jsonFactory = new JsonFactory();

		int relCount = 0;

		if(subject.hasProperty(RDF.type, CRMpe.PE1_Service)){

			StmtIterator itOfferedBy = subject.listProperties(CRMpe.PP1i_is_currently_offered_by);

			while(itOfferedBy.hasNext()){

				Resource project = itOfferedBy.nextStatement().getResource();

				ParthenosRegistryResource source = idMap.get(uri);

				ParthenosRegistryResource projectRes = idMap.get(project.getURI());

				String relJson = relWriter.writeRelationship(jsonFactory, "PP1_currently_offers", projectRes.getUuid(), projectRes.getType(), source.getUuid(), source.getType());

				this.resourceRegistryPublisher.createIsRelatedTo("PP1_currently_offers", relJson);

				relCount++;

			}

			StmtIterator itProvidedBy = subject.listProperties(CRMpe.PP2_provided_by);

			while(itProvidedBy.hasNext()){

				Resource provider = itProvidedBy.nextStatement().getResource();

				ParthenosRegistryResource prr = idMap.get(uri);

				ParthenosRegistryResource providerRes = idMap.get(provider.getURI());

				String relJson = relWriter.writeRelationship(jsonFactory, "PP2_provided_by", prr.getUuid(), prr.getType(), providerRes.getUuid(), providerRes.getType());

				this.resourceRegistryPublisher.createIsRelatedTo("PP2_provided_by", relJson);

				relCount++;

				//TODO: IsRelatedTo the contact person of the provider. Interpret '"Follow path of service ‘Provided by’ and switch E39 for E21: E21- >p76->E51""

			}

			relCount += registerRelationship(CRMpe.PP45_has_competency, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP4_hosts_object, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP31_uses_curation_plan, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP32_curates, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP6_hosts_digital_object, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP7_hosts_software_object, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP8_hosts_dataset, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP29_uses_access_protocol, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP47_has_protocol_type, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP48_uses_protocol_parameter, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP46_brokers_access_to, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP11_curates_volatile_digital_object, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP12_curates_volatile_software, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP13_curates_volatile_dataset, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP15_delivers_on_request, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP46_brokers_access_to, subject, idMap, jsonFactory);

		}

		if(subject.hasProperty(RDF.type, CRMpe.PE18_Dataset)){

			StmtIterator itPartOf = subject.listProperties(CRMpe.PP23i_is_dataset_part_of);

			while(itPartOf.hasNext()){

				Resource obj = itPartOf.nextStatement().getResource();

				ParthenosRegistryResource prr = idMap.get(uri);

				ParthenosRegistryResource objres = idMap.get(obj.getURI());

				String relJson = relWriter.writeRelationship(jsonFactory, "PP23_has_dataset_part", objres.getUuid(), objres.getType(), prr.getUuid(), prr.getType());

				this.resourceRegistryPublisher.createIsRelatedTo("PP23_has_dataset_part", relJson);

				relCount++;

			}

			//TODO: this is in the registry model but I am not sure what's going to happen...

			StmtIterator itIsAbout = subject.listProperties(CRM.P129_is_about);

			while(itIsAbout.hasNext()){

				Resource obj = itIsAbout.nextStatement().getResource();

				if(obj.hasProperty(RDF.type, CRM.E55_Type)) {

					ParthenosRegistryResource prr = idMap.get(uri);

					ParthenosRegistryResource objres = idMap.get(obj.getURI());

					String relJson = relWriter.writeRelationship(jsonFactory, "P129_is_about", prr.getUuid(), prr.getType(), objres.getUuid(), objres.getType());

					this.resourceRegistryPublisher.createIsRelatedTo("P129_is_about", relJson);

					relCount++;

				}

			}

		}

		if(subject.hasProperty(RDF.type, CRMpe.PE19_Persistent_Digital_Object)){

			relCount += registerRelationship(CRMpe.PP16_has_persistent_digital_object_part, subject, idMap, jsonFactory);

		}

		if(subject.hasProperty(RDF.type, CRMpe.PE20_Volatile_Digital_Object)){

			relCount += registerRelationship(CRMpe.PP17_has_snapshot, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP18_has_digital_object_part, subject, idMap, jsonFactory);

		}

		if(subject.hasProperty(RDF.type, CRMpe.PE21_Persistent_Software)){

			relCount += registerRelationship(CRMpe.PP19_has_persistent_software_part, subject, idMap, jsonFactory);

		}

		if(subject.hasProperty(RDF.type, CRMpe.PE22_Persistent_Dataset)){

			relCount += registerRelationship(CRMpe.PP20_has_persistent_dataset_part, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP39_is_metadata_for, subject, idMap, jsonFactory);

		}

		if(subject.hasProperty(RDF.type, CRMpe.PE23_Volatile_Software)){

			relCount += registerRelationship(CRMpe.PP21_has_software_part, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP22_has_release, subject, idMap, jsonFactory);

		}

		if(subject.hasProperty(RDF.type, CRMpe.PE24_Volatile_Dataset)){

			relCount += registerRelationship(CRMpe.PP23_has_dataset_part, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP24_has_dataset_snapshot, subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP41_is_index_of, subject, idMap, jsonFactory);

		}

		if(subject.hasProperty(RDF.type, CRMpe.PE26_RI_Project)){

			relCount += registerRelationship(CRMpe.PP25_has_maintaining_RI, subject, idMap, jsonFactory);

		}

		if(subject.hasProperty(RDF.type, CRMpe.PE35_Project)){

			relCount += registerRelationship(CRMpe.PP43_supported_project_activity,  subject, idMap, jsonFactory);

			relCount += registerRelationship(CRMpe.PP44_has_maintaining_team, subject, idMap, jsonFactory);

		}

		log.debug(String.format("Registered %d relationships for uri %s, uuid %s", relCount, uri, uuid));

		return relCount;

	}

	protected int registerRelationship(final Property rel, final Resource subject, final Map<String,ParthenosRegistryResource> idMap, final JsonFactory jsonFactory)

			throws IOException, ResourceRegistryException, ParthenosPublisherException {

		int relCount = 0;

		StmtIterator it = subject.listProperties(rel);

		while(it.hasNext()){

			Resource obj = it.nextStatement().getResource();

			if(obj == null) throw new ParthenosPublisherException(String.format("No target resource available for %s --> %s", subject.getURI(), rel.getURI()));

			ParthenosRegistryResource source = idMap.get(subject.getURI());

			ParthenosRegistryResource target = idMap.get(obj.getURI());

			String relJson = relWriter.writeRelationship(jsonFactory, rel.getLocalName(), source.getUuid(), source.getType(), target.getUuid(), target.getType());

			this.resourceRegistryPublisher.createIsRelatedTo(rel.getLocalName(), relJson);

			relCount++;

		}

		return relCount;

	}

	protected InfModel loadBaseModel() {

		return ModelFactory.createRDFSModel(baseModel);

	}

	protected int processServices(final InfModel model, final Set<String> uriProcessed, final JsonFactory jsonFactory, final Map<String, ParthenosRegistryResource> idMap)

			throws IOException, ResourceRegistryException {

		log.debug("Processing services");

		int count = 0;

		int total = 0;

		ResIterator iter = model.listResourcesWithProperty(RDF.type, CRMpe.PE1_Service);

		while (iter.hasNext()) {

			total++;

			Resource res = iter.nextResource();

			String resourceURI = res.getURI();

			if (!uriProcessed.contains(resourceURI)) {

				ParthenosRegistryResource prr = processService(res, jsonFactory, idMap);

				this.resourceRegistryPublisher.createResource(prr.getType(), prr.getJson());

				uriProcessed.add(resourceURI);

				idMap.put(resourceURI, prr);

				count++;

			} else {

				log.debug(resourceURI + " already processed, now skipping it");

			}

		}

		log.debug(String.format("Registered %d/%d services", count, total));

		return count;

	}

	private ParthenosRegistryResource processService(final Resource res, final JsonFactory jsonFactory, final Map<String, ParthenosRegistryResource> idMap) throws IOException {

		log.debug("Processing " + res.getURI());

		StringWriter sw = new StringWriter();

		JsonGenerator jg = jsonFactory.createGenerator(sw);

		jg.writeStartObject();

		String specificType = findSpecificType(res, CRMpe.PE1_Service).getLocalName();

		String uuid = writeCommon(res, specificType, jg, idMap);

		//******THE FACETS *******//

		jg.writeArrayFieldStart("consistsOf");

		//list of facets

		writeCommonFacets(res, jg);

		facetWriter.writeEventFacet(jg);

		facetWriter.writeRightsFacet(jg, res);

		if (res.hasProperty(CRMpe.PP2_provided_by)) {

			//TODO: shouldn't this be a rel to an actor?

			Resource provider = res.getPropertyResourceValue(CRMpe.PP2_provided_by);

			facetWriter.writeContactReferenceFacet(jg, provider);

		}

		facetWriter.writeDesignatedAccessPointFacet(jg, res);

		jg.writeEndArray();

		jg.writeEndObject();

		jg.close();

		String json = sw.toString();

		log.debug(json);

		return new ParthenosRegistryResource().setJson(json).setType(specificType).setUuid(uuid);

	}

	/**

	 * Write the common properties: header, class for all resources

	 *

	 * @param resource  input Resource.

	 * @param className name of the class for the registry

	 * @param jg        JsonGenerator to write with.

	 * @param idMap     map of the already assigned UUID

	 * @return the uuid of the resource

	 */

	protected String writeCommon(final Resource resource, final String className, final JsonGenerator jg, final Map<String, ParthenosRegistryResource> idMap) throws IOException {

		String uuid;

		if (idMap.containsKey(resource.getURI())) {

			uuid = idMap.get(resource.getURI()).getUuid();

		} else {

			uuid = UUID.randomUUID().toString();

			idMap.put(resource.getURI(), new ParthenosRegistryResource().setUuid(uuid));

		}

		jg.writeStartObject();

		jg.writeObjectFieldStart("header");

		jg.writeStringField("uuid", uuid);

		jg.writeEndObject();

		jg.writeStringField("@class", className);

		return uuid;

	}

	/**

	 * Write the common facets: identifier and info facets

	 *

	 * @param res

	 * @param jg

	 */

	protected void writeCommonFacets(final Resource res, final JsonGenerator jg) throws IOException {

		facetWriter.writeIdentifierFacet(jg, res.getURI());

		facetWriter.writeP1Facets(jg, res);

		facetWriter.writeInfoFacet(jg, res);

	}

	protected int processDigitalObjects(final InfModel model, final Set<String> uriProcessed, final JsonFactory jsonFactory, final Map<String, ParthenosRegistryResource> idMap)

			throws IOException, ResourceRegistryException {

		log.debug("Processing digital objects (include dataset and software)");

		int count = 0;

		int total = 0;

		ResIterator iterDo = model.listResourcesWithProperty(RDF.type, CRMdig.D1_Digital_Object);

		while (iterDo.hasNext()) {

			total++;

			Resource res = iterDo.nextResource();

			String resourceURI = res.getURI();

			if (!uriProcessed.contains(resourceURI)) {

				ParthenosRegistryResource prr = processDigitalObject(res, jsonFactory, idMap);

				this.resourceRegistryPublisher.createResource(prr.getType(), prr.getJson());

				uriProcessed.add(resourceURI);

				idMap.put(resourceURI, prr);

				count++;

			} else {

				log.debug(resourceURI + " already processed, now skipping it");

			}

		}

		log.debug(String.format("Registered %d/%d digital objects", count, total));

		return count;

	}

	protected ParthenosRegistryResource processDigitalObject(final Resource res, final JsonFactory jsonFactory, final Map<String, ParthenosRegistryResource> idMap)

			throws IOException {

		log.debug("Processing " + res.getURI());

		String specificType = findSpecificType(res, CRMdig.D1_Digital_Object).getLocalName();

		StringWriter sw = new StringWriter();

		JsonGenerator jg = jsonFactory.createGenerator(sw);

		jg.writeStartObject();

		String uuid = writeCommon(res, specificType, jg, idMap);

		//******THE FACETS *******//

		jg.writeArrayFieldStart("consistsOf");

		//list of facets

		writeCommonFacets(res, jg);

		facetWriter.writeTemporalCoverageFacet(jg, res);

		facetWriter.writeRightsFacet(jg, res);

		jg.writeEndArray();

		jg.writeEndObject();

		jg.close();

		String json = sw.toString();

		log.debug(json);

		return new ParthenosRegistryResource().setUuid(uuid).setType(specificType).setJson(json);

	}

	protected int processActors(final InfModel model, Set<String> uriProcessed, final JsonFactory jsonFactory, final Map<String, ParthenosRegistryResource> idMap)

			throws IOException {

		log.debug("Processing actors");

		int count = 0;

		int total = 0;

		ResIterator iterActors = model.listResourcesWithProperty(RDF.type, CRM.E39_Actor);

		while (iterActors.hasNext()) {

			total++;

			Resource res = iterActors.nextResource();

			String resourceURI = res.getURI();

			if (!uriProcessed.contains(resourceURI)) {

				ParthenosRegistryResource prr = processActor(res, jsonFactory, idMap);

				uriProcessed.add(resourceURI);

				idMap.put(resourceURI, prr);

				count++;

			} else {

				log.debug(resourceURI + " already processed, now skipping it");

			}

		}

		log.debug(String.format("Registered %d/%d actors", count, total));

		return count;

	}

	protected ParthenosRegistryResource processActor(final Resource r, final JsonFactory jsonFactory, final Map<String, ParthenosRegistryResource> idMap) throws IOException {

		log.debug("Processing " + r.getURI());

		String specificType = findSpecificType(r, CRM.E39_Actor).getLocalName();

		StringWriter sw = new StringWriter();

		JsonGenerator jg = jsonFactory.createGenerator(sw);

		jg.writeStartObject();

		String uuid = writeCommon(r, specificType, jg, idMap);

		//******THE FACETS *******//

		jg.writeArrayFieldStart("consistsOf");

		//list of facets

		writeCommonFacets(r, jg);

		facetWriter.writeContactReferenceFacet(jg, r);

		jg.writeEndArray();

		jg.writeEndObject();

		jg.close();

		String json = sw.toString();

		log.debug(json);

		return new ParthenosRegistryResource().setJson(json).setType(specificType).setUuid(uuid);

	}

	protected int processProjects(final InfModel model, Set<String> uriProcessed, final JsonFactory jsonFactory, final Map<String, ParthenosRegistryResource> idMap)

			throws IOException, ResourceRegistryException {

		log.debug("Processing projects");

		int count = 0;

		int total = 0;

		ResIterator iterProjects = model.listResourcesWithProperty(RDF.type, CRMpe.PE35_Project);

		while (iterProjects.hasNext()) {

			total++;

			Resource res = iterProjects.nextResource();

			String resourceURI = res.getURI();

			if (!uriProcessed.contains(resourceURI)) {

				ParthenosRegistryResource prr = processProject(res, jsonFactory, idMap);

				this.resourceRegistryPublisher.createResource(prr.getType(), prr.getJson());

				uriProcessed.add(resourceURI);

				idMap.put(resourceURI, prr);

				count++;

			} else {

				log.debug(resourceURI + " already processed, now skipping it");

			}

		}

		log.debug(String.format("Registered %d/%d projects", count, total));

		return count;

	}

	protected ParthenosRegistryResource processProject(final Resource res, final JsonFactory jsonFactory, final Map<String, ParthenosRegistryResource> idMap) throws IOException {

		log.debug("Processing " + res.getURI());

		String specificType = findSpecificType(res, CRMpe.PE35_Project).getLocalName();

		StringWriter sw = new StringWriter();

		JsonGenerator jg = jsonFactory.createGenerator(sw);

		jg.writeStartObject();

		String uuid = writeCommon(res, specificType, jg, idMap);

		jg.writeArrayFieldStart("consistsOf");

		writeCommonFacets(res, jg);

		jg.writeEndArray();

		jg.writeEndObject();

		jg.close();

		String json = sw.toString();

		log.debug(json);

		return new ParthenosRegistryResource().setJson(json).setType(specificType).setUuid(uuid);

	}

	/**

	 * Finds the most specific type of res.

	 *

	 * @param res          Resource you want to find the most specific type

	 * @param fallbackType Resource representing the type to return if there is no type or if we get AmbiguousSpecificTypeException

	 * @return Resource: the most specific type, if any. fallbackType otherwise

	 */

	protected Resource findSpecificType(final Resource res, final Resource fallbackType) {

		Resource specType = fallbackType;

		try {

			specType = AssemblerHelp.findSpecificType(res, fallbackType);

		} catch (AmbiguousSpecificTypeException e) {

			log.warn(res.getURI() + ": " + e.getMessage());

		}

		return specType;

	}

	public FacetWriter getFacetWriter() {

		return facetWriter;

	}

	public void setFacetWriter(final FacetWriter facetWriter) {

		this.facetWriter = facetWriter;

	}

}