D-Net

package eu.dnetlib.data.transform.xml2;

import java.util.List;

import java.util.Map;

import java.util.Map.Entry;

import java.util.Objects;

import java.util.function.Function;

import com.google.common.collect.Streams;

import com.google.protobuf.Descriptors.Descriptor;

import com.ximpleware.AutoPilot;

import com.ximpleware.VTDGen;

import com.ximpleware.VTDNav;

import eu.dnetlib.data.proto.FieldTypeProtos;

import eu.dnetlib.data.proto.FieldTypeProtos.*;

import eu.dnetlib.data.proto.FieldTypeProtos.OAIProvenance.OriginDescription;

import eu.dnetlib.data.proto.KindProtos.Kind;

import eu.dnetlib.data.proto.OafProtos.Oaf;

import eu.dnetlib.data.proto.OafProtos.OafEntity;

import eu.dnetlib.data.proto.OafProtos.OafRel;

import eu.dnetlib.data.proto.RelTypeProtos.RelType;

import eu.dnetlib.data.proto.RelTypeProtos.SubRelType;

import eu.dnetlib.data.proto.ResultProtos.Result;

import eu.dnetlib.data.proto.ResultProtos.Result.*;

import eu.dnetlib.data.proto.TypeProtos.Type;

import eu.dnetlib.miscutils.collections.Pair;

import eu.dnetlib.pace.model.Person;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.lang3.exception.ExceptionUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import static eu.dnetlib.data.transform.xml.AbstractDNetXsltFunctions.oafSimpleId;

import static eu.dnetlib.data.transform.xml.AbstractDNetXsltFunctions.oafSplitId;

import static eu.dnetlib.data.transform.xml2.Utils.*;

import static eu.dnetlib.data.transform.xml2.VtdUtilityParser.*;

import static java.lang.String.format;

public abstract class AbstractResultVtdParser implements Function<String, Oaf> {

	private static final Log log = LogFactory.getLog(AbstractResultVtdParser.class);

	protected boolean invisible = false;

	protected String provenance = "";

	protected String trust = "0.9";

	protected SpecificationMap specs;

	public AbstractResultVtdParser(final Map<String, String> fields) {

		this.specs = buildSpecs(fields);

	}

	public AbstractResultVtdParser(final boolean invisible, final String provenance, final String trust, final Map<String, String> fields) {

		this(fields);

		this.invisible = invisible;

		this.provenance = provenance;

		this.trust = trust;

	}

	protected abstract String getResulttype(final String cobjcategory);

	@Override

	public Oaf apply(final String xml) {

		try {

			final VTDGen vg = parseXml(xml);

			final VTDNav vn = vg.getNav();

			final AutoPilot ap = new AutoPilot(vn);

			final boolean skiprecord = Boolean.valueOf(getFirstValue(ap, vn, xpath("record", "header", "skipRecord")));

			int metadata = countNodes(ap, vn, format("count(%s)", xpath("record", "metadata")));

			if (metadata == 0 || skiprecord) {

				return null;

			}

			final String objIdentifier = oafSimpleId(Type.result.name(), getFirstValue(ap, vn, xpath("record", "header", "objIdentifier")));

			if (StringUtils.isBlank(objIdentifier)) {

				return null;

			}

			for(final Entry<Descriptor, SpecificationDescriptor> spec : specs.entrySet()) {

				final Descriptor d = spec.getKey();

				final SpecificationDescriptor md = spec.getValue();

				for(Entry<String, Pair<String, Function<List<Node>, Object>>> entry : md.getFields().entrySet()) {

					final String fieldName = entry.getKey();

					final Pair<String, Function<List<Node>, Object>> pair = entry.getValue();

					final String xpath = pair.getKey();

					final Function<List<Node>, Object> function = pair.getValue();

					try {

						addField(md.getBuilder(), d.findFieldByName(fieldName), function.apply(getNodes(ap, vn, xpath)));

					} catch (Throwable e) {

						throw new VtdException(String.format("Error mapping field '%s' from xpath '%s' for record '%s'", fieldName, xpath, objIdentifier), e);

					}

				}

			}

			return Oaf.newBuilder()

					.setKind(Kind.entity)

					.setDataInfo(ensureDataInfo(ap, vn, DataInfo.newBuilder()))

					.setEntity(((OafEntity.Builder) specs.get(OafEntity.getDescriptor())

							.getBuilder()

							.setField(

									OafEntity.getDescriptor().findFieldByName(Type.result.name()),

									((Result.Builder) specs.get(Result.getDescriptor()).getBuilder())

											.setMetadata((Metadata) specs.get(Metadata.getDescriptor()).getBuilder().build())

											.addInstance((Instance) specs.get(Instance.getDescriptor()).getBuilder().build())

											.build()))

							.setId(objIdentifier)

							.setOaiprovenance(getOaiProvenance(ap, vn))

							.build())

					.build();

		} catch (Throwable e) {

			log.error(xml);

			log.error(ExceptionUtils.getStackTrace(e));

			return null;

		}

	}

	public SpecificationMap buildSpecs(final Map<String, String> fields) {

		final SpecificationMap specs = new SpecificationMap();

		specs.put(Result.getDescriptor(), SpecificationDescriptor.newInstance())

				.setBuilder(Result.newBuilder())

				.put("externalReference", fields.get("externalReference"), nodes -> nodes.stream()

						.map(node -> {

							final ExternalReference.Builder extref = ExternalReference.newBuilder();

							if (StringUtils.isNotBlank(node.getTextValue())) {

								extref.setUrl(node.getTextValue());

							}

							final Map<String, String> a = node.getAttributes();

							final String source = a.get("source");

							if (StringUtils.isNotBlank(source)) {

								extref.setSitename(source);

							}

							final String identifier = a.get("identifier");

							if (StringUtils.isNotBlank(identifier)) {

								extref.setRefidentifier(identifier);

							}

							final String title = a.get("title");

							if (StringUtils.isNotBlank(title)) {

								extref.setLabel(title);

							}

							final String query = a.get("query");

							if (StringUtils.isNotBlank(query)) {

								extref.setQuery(query);

							}

							final String type = a.get("type");

							if (StringUtils.isNotBlank(type)) {

								extref.setQualifier(getSimpleQualifier(type, DNET_EXT_REF_TYPOLOGIES));

							}

							return extref.build();

						}));

		specs.put(Instance.getDescriptor(), SpecificationDescriptor.newInstance())

				.setBuilder(Instance.newBuilder())

				.put("license", fields.get("license"), nodes -> nodes.stream()

						.filter(node -> {

							final Map<String, String> a = node.getAttributes();

							switch (node.getName()) {

							case "rights":

								return a.containsKey(RIGHTS_URI) && a.get(RIGHTS_URI).matches(URL_REGEX);

							case "license":

								return true;

							default:

								return false;

							}

						})

						.map(Node::getTextValue))

				.put("accessright", fields.get("accessright"), nodes -> nodes.stream()

						.map(Node::getTextValue)

						.map(rights -> mappingAccess.containsKey(rights) ? mappingAccess.get(rights) : "UNKNOWN")

						.map(code -> getQualifier(code, getClassName(code), DNET_ACCESS_MODES, DNET_ACCESS_MODES)))

				.put("instancetype", fields.get("instancetype"), nodes -> nodes.stream()

						.map(Node::getTextValue)

						.map(code -> getQualifier(code, getClassName(code), DNET_PUBLICATION_RESOURCE, DNET_PUBLICATION_RESOURCE)))

				.put("hostedby", fields.get("hostedby"), nodes -> nodes.stream()

						.map(node -> getKV(oafSplitId("datasource", node.getAttributes().get("id")), node.getAttributes().get("name"))))

				.put("url", fields.get("url"), nodes -> nodes.stream()

						.map(Node::getTextValue)

						.filter(s -> s.trim().matches(URL_REGEX)))

				.put("dateofacceptance", fields.get("dateofacceptance"), nodes -> nodes.stream()

						.map(Node::getTextValue));

		specs.put(Metadata.getDescriptor(), SpecificationDescriptor.newInstance())

				.setBuilder(Metadata.newBuilder())

				.put("title", fields.get("title"), nodes -> nodes.stream()

						.map(node -> {

							final Qualifier.Builder q = Qualifier.newBuilder().setSchemeid(DNET_TITLE_TYPOLOGIES).setSchemename(DNET_TITLE_TYPOLOGIES);

							switch (node.getAttributes().get(TITLE_TYPE) + "") {

							case "AlternativeTitle":

								q.setClassid("alternative title").setClassname("alternative title");

								break;

							case "Subtitle":

								q.setClassid("subtitle").setClassname("subtitle");

								break;

							case "TranslatedTitle":

								q.setClassid("translated title").setClassname("translated title");

								break;

							default:

								q.setClassid("main title").setClassname("main title");

								break;

							}

							return StructuredProperty.newBuilder().setValue(node.getTextValue()).setQualifier(q).build();

						}))

				.put("description", fields.get("description"), nodes -> nodes.stream()

						.map(Node::getTextValue))

				.put("storagedate", fields.get("storagedate"), nodes -> nodes.stream()

						.map(Node::getTextValue))

				.put("lastmetadataupdate", fields.get("lastmetadataupdate"), nodes -> nodes.stream()

						.map(Node::getTextValue))

				.put("embargoenddate", fields.get("embargoenddate"), nodes -> nodes.stream()

						.map(Node::getTextValue))

				.put("dateofacceptance", fields.get("dateofacceptance"), nodes -> nodes.stream()

						.map(Node::getTextValue))

				.put("author", fields.get("author"), nodes -> Streams.mapWithIndex(

						nodes.stream()

								.map(Node::getTextValue),

						(creator, i) -> new Pair<>(i, creator))

						.map(pair -> {

							final Author.Builder author = Author.newBuilder();

							author.setFullname(pair.getValue());

							author.setRank(pair.getKey().intValue() + 1);

							final Person p = new Person(pair.getValue(), false);

							if (p.isAccurate()) {

								author.setName(p.getNormalisedFirstName());

								author.setSurname(p.getNormalisedSurname());

							}

							return author.build();

						}))

				.put("contributor", fields.get("contributor"), nodes -> nodes.stream()

						.map(Node::getTextValue))

				.put("subject", fields.get("subject"), nodes -> nodes.stream()

						.map(node -> {

							final Map<String, String> a = node.getAttributes();

							final String classId = StringUtils.isNotBlank(a.get(CLASSID)) ? a.get(CLASSID) : KEYWORD;

							final String className = StringUtils.isNotBlank(a.get(CLASSNAME)) ? a.get(CLASSNAME) : KEYWORD;

							final String schemeId = StringUtils.isNotBlank(a.get(SCHEMEID)) ? a.get(SCHEMEID) : DNET_SUBJECT_TYPOLOGIES;

							final String schemeName = StringUtils.isNotBlank(a.get(SCHEMENAME)) ? a.get(SCHEMENAME) : DNET_SUBJECT_TYPOLOGIES;

							return getStructuredProperty(node.getTextValue(), classId, className, schemeId, schemeName);

						}))

				.put("format", fields.get("format"), nodes -> nodes.stream()

						.map(Node::getTextValue))

				.put("source", fields.get("source"), nodes -> nodes.stream()

						.map(Node::getTextValue))

				.put("size", fields.get("size"), nodes -> nodes.stream()

						.map(Node::getTextValue))

				.put("version", fields.get("version"), nodes -> nodes.stream()

						.map(Node::getTextValue))

				.put("publisher", fields.get("publisher"), nodes -> nodes.stream()

						.map(Node::getTextValue))

				.put("language", fields.get("language"), nodes -> nodes.stream()

						.map(Node::getTextValue)

						.map(code -> getQualifier(code, getClassName(code), DNET_LANGUAGES, DNET_LANGUAGES)))

				.put("resourcetype", fields.get("resourcetype"), nodes -> nodes.stream()

						.map(node -> node.getAttributes().get("resourceTypeGeneral"))

						.map(resourceType -> getSimpleQualifier(resourceType, DNET_DATA_CITE_RESOURCE)))

				.put("resulttype", fields.get("resulttype"), nodes -> nodes.stream()

						.map(Node::getTextValue)

						.map(cobjcategory -> getSimpleQualifier(getResulttype(cobjcategory), DNET_RESULT_TYPOLOGIES)))

				.put("concept", fields.get("concept"), nodes -> nodes.stream()

						.filter(node -> node.getAttributes() != null && StringUtils.isNotBlank(node.getAttributes().get("id")))

						.map(node -> Context.newBuilder().setId(node.getAttributes().get("id"))))

				.put("journal", fields.get("journal"), nodes -> nodes.stream()

						.map(node -> {

							final Journal.Builder journal = Journal.newBuilder();

							if (StringUtils.isNotBlank(node.getTextValue())) {

								journal.setName(node.getTextValue());

							}

							if (node.getAttributes() != null) {

								final Map<String, String> a = node.getAttributes();

								if (StringUtils.isNotBlank(a.get("issn"))) {

									journal.setIssnPrinted(a.get("issn"));

								}

								if (StringUtils.isNotBlank(a.get("eissn"))) {

									journal.setIssnOnline(a.get("eissn"));

								}

								if (StringUtils.isNotBlank(a.get("lissn"))) {

									journal.setIssnLinking(a.get("lissn"));

								}

								if (StringUtils.isNotBlank(a.get("sp"))) {

									journal.setSp(a.get("sp"));

								}

								if (StringUtils.isNotBlank(a.get("ep"))) {

									journal.setEp(a.get("ep"));

								}

								if (StringUtils.isNotBlank(a.get("iss"))) {

									journal.setIss(a.get("iss"));

								}

								if (StringUtils.isNotBlank(a.get("vol"))) {

									journal.setVol(a.get("vol"));

								}

							}

							return journal;

						}));

		specs.put(OafEntity.getDescriptor(), SpecificationDescriptor.newInstance())

				.setBuilder(OafEntity.newBuilder().setType(Type.result))

				.put("originalId", fields.get("originalId"), nodes -> nodes.stream()

						.map(Node::getTextValue)

						.map(s -> StringUtils.contains(s, ID_SEPARATOR) ? StringUtils.substringAfter(s, ID_SEPARATOR) : s)

						.filter(s -> !s.trim().matches(URL_REGEX)))

				.put("collectedfrom", fields.get("collectedfrom"), nodes -> nodes.stream()

						.map(node -> getKV(

								oafSplitId(Type.datasource.name(), node.getAttributes().get("id")),

								node.getAttributes().get("name"))))

				.put("pid", fields.get("pid"), nodes -> nodes.stream()

						.filter(pid -> {

							final Map<String, String> a = pid.getAttributes();

							return a.containsKey(IDENTIFIER_TYPE) || a.containsKey(ALTERNATE_IDENTIFIER_TYPE);

						})

						.filter(pid -> {

							final Map<String, String> a = pid.getAttributes();

							return !"url".equalsIgnoreCase(a.get(IDENTIFIER_TYPE)) && !"url".equalsIgnoreCase(a.get(ALTERNATE_IDENTIFIER_TYPE));

						})

						.map(pid -> {

							final Map<String, String> a = pid.getAttributes();

							final String identifierType = a.get(IDENTIFIER_TYPE);

							final String altIdentifierType = a.get(ALTERNATE_IDENTIFIER_TYPE);

							return StructuredProperty.newBuilder()

									.setValue(pid.getTextValue())

									.setQualifier(getSimpleQualifier(

											StringUtils.isNotBlank(identifierType) ?

													identifierType : altIdentifierType, DNET_PID_TYPES))

									.build();

						}))

				.put("dateofcollection", fields.get("dateofcollection"), nodes -> nodes.stream()

						.map(Node::getTextValue))

				.put("dateoftransformation", fields.get("dateoftransformation"), nodes -> nodes.stream()

						.map(Node::getTextValue))

				.put("cachedRel", fields.get("cachedRel"), nodes -> nodes.stream()

						.map(node -> getOafRel(node,

								OafRel.newBuilder()

										.setSource("")

										.setChild(false)))

						.filter(Objects::nonNull)

						.map(oafRel -> oafRel.build()));

		return specs;

	}

	private static OafRel.Builder getOafRel(final Node node, final OafRel.Builder oafRel) {

		final Map<String, String> a = node.getAttributes();

		switch (node.getName()) {

		case PROJECTID:

			if (StringUtils.isBlank(node.getTextValue())) {

				return null;

			}

			return oafRel

					.setTarget(oafSplitId(Type.project.name(), StringUtils.trim(node.getTextValue())))

					.setRelType(RelType.resultProject)

					.setSubRelType(SubRelType.outcome)

					.setRelClass("isProducedBy");

		case RELATED_PUBLICATION:

		case RELATED_DATASET:

			if (StringUtils.isBlank(a.get("id"))) {

				return null;

			}

			return oafRel

					.setTarget(oafSimpleId(Type.result.name(), StringUtils.trim(a.get("id"))))

					.setRelType(RelType.resultResult)

					.setSubRelType(SubRelType.publicationDataset)

					.setRelClass("isRelatedTo");

		case RELATED_IDENTIFIER:

			if (StringUtils.isBlank(node.getTextValue())) {

				return null;

			}

			return oafRel

					.setTarget(node.getTextValue())

					.setRelType(RelType.resultResult)

					.setSubRelType(SubRelType.relationship)

					.setRelClass(a.get(RELATION_TYPE))

					.setCachedTarget(

							OafEntity.newBuilder()

									.setType(Type.result)

									.setId("") //TODO

									.addPid(

											StructuredProperty.newBuilder()

													.setValue(node.getTextValue())

													.setQualifier(getSimpleQualifier(a.get(RELATED_IDENTIFIER_TYPE), DNET_PID_TYPES))

													.build()));

		default:

			return null;

		}

	}

	private OriginDescription getOriginDescription(final AutoPilot ap, final VTDNav vn, final String basePath) throws VtdException {

		final OriginDescription.Builder od = OriginDescription.newBuilder();

		if (getNodes(ap, vn, basePath).isEmpty()) {

			return od.build();

		}

		final Map<String, String> odAttr = getNode(ap, vn, basePath).getAttributes();

		final String harvestDate = odAttr.get("harvestDate");

		if (StringUtils.isNotBlank(harvestDate)) {

			od.setHarvestDate(harvestDate);

		}

		final String altered = odAttr.get("altered");

		if (StringUtils.isNotBlank(altered)) {

			od.setAltered(Boolean.valueOf(altered));

		}

		final String baseUrl = getFirstValue(ap, vn, basePath + xpath("baseURL"));

		if (StringUtils.isNotBlank(basePath)) {

			od.setBaseURL(baseUrl);

		}

		final String identifier = getFirstValue(ap, vn, basePath + xpath("identifier"));

		if (StringUtils.isNotBlank(identifier)) {

			od.setIdentifier(identifier);

		}

		final String datestamp = getFirstValue(ap, vn, basePath + xpath("datestamp"));

		if (StringUtils.isNotBlank(datestamp)) {

			od.setDatestamp(datestamp);

		}

		final String metadataNamespace = getFirstValue(ap, vn, basePath + xpath("metadataNamespace"));

		if (StringUtils.isNotBlank(metadataNamespace)) {

			od.setMetadataNamespace(metadataNamespace);

		}

		final OriginDescription originDescription = getOriginDescription(ap, vn, basePath + xpath("originDescription"));

		if (originDescription.hasHarvestDate()) {

			od.setOriginDescription(originDescription);

		}

		return od.build();

	}

	private OAIProvenance getOaiProvenance(final AutoPilot ap, final VTDNav vn) throws VtdException {

		return OAIProvenance.newBuilder()

				.setOriginDescription(getOriginDescription(ap, vn, xpath("record", "about", "provenance", "originDescription")))

				.build();

	}

	private FieldTypeProtos.DataInfo.Builder ensureDataInfo(

    		final AutoPilot ap, final VTDNav vn,

            final DataInfo.Builder info) throws VtdException {

        if (info.isInitialized()) return info;

        return buildDataInfo( ap, vn, invisible, provenance, trust, false, false);

    }

	private FieldTypeProtos.DataInfo.Builder buildDataInfo(

            final AutoPilot ap,

            final VTDNav vn,

            final boolean invisible,

            final String defaultProvenanceaction,

            final String defaultTrust,

            final boolean defaultDeletedbyinference,

            final boolean defaultInferred) throws VtdException {

		final DataInfo.Builder dataInfoBuilder = FieldTypeProtos.DataInfo.newBuilder()

            .setInvisible(invisible)

			.setInferred(defaultInferred)

            .setDeletedbyinference(defaultDeletedbyinference)

            .setTrust(defaultTrust)

	        .setProvenanceaction(getSimpleQualifier(defaultProvenanceaction, DNET_PROVENANCE_ACTIONS));

        // checking instanceof because when receiving an empty <oaf:datainfo> we don't want to parse it.

	    final String xpath = xpath("record", "about", "datainfo");

	    if (getNodes(ap, vn, xpath).size() > 0) {

		    final Map<String, String> provAction = getNode(ap, vn, xpath + xpath("provenanceaction")).getAttributes();

		    dataInfoBuilder

				    .setInvisible(Boolean.valueOf(getValue(getNode(ap, vn, xpath + xpath("invisible")), String.valueOf(invisible))))

				    .setInferred(Boolean.valueOf(getValue(getNode(ap, vn, xpath + xpath("inferred")), String.valueOf(defaultInferred))))

				    .setDeletedbyinference(Boolean.valueOf(

						    getValue(getNode(ap, vn, xpath + xpath("deletedbyinference")), String.valueOf(defaultDeletedbyinference))))

				    .setTrust(getValue(getNode(ap, vn, xpath + xpath("trust")), defaultTrust))

				    .setInferenceprovenance(getValue(getNode(ap, vn, xpath + xpath("inferenceprovenance")), ""))

				    .setProvenanceaction(getSimpleQualifier(

						    getValue(provAction.get(CLASSID), defaultProvenanceaction),

						    DNET_PROVENANCE_ACTIONS));

	    }

	    return dataInfoBuilder;

    }

}