D-Net

package eu.dnetlib.data.collective.transformation.engine;

import java.io.StringReader;

import java.util.Iterator;

import java.util.LinkedHashMap;

import java.util.LinkedList;

import java.util.List;

import java.util.Map;

import javax.xml.xpath.XPath;

import javax.xml.xpath.XPathConstants;

import javax.xml.xpath.XPathExpressionException;

import javax.xml.xpath.XPathFactory;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.core.io.Resource;

import org.w3c.dom.Node;

import org.xml.sax.InputSource;

import eu.dnetlib.common.profile.ResourceDao;

import eu.dnetlib.data.collective.transformation.IDatabaseConnector;

import eu.dnetlib.data.collective.transformation.TransformationException;

import eu.dnetlib.data.collective.transformation.VocabularyRegistry;

import eu.dnetlib.data.collective.transformation.core.xsl.ext.TransformationFunctionProxy;

import eu.dnetlib.data.collective.transformation.engine.core.ITransformation;

import eu.dnetlib.data.collective.transformation.engine.functions.Convert;

// import eu.dnetlib.data.collective.transformation.engine.functions.Dblookup;

import eu.dnetlib.data.collective.transformation.engine.functions.Extract;

import eu.dnetlib.data.collective.transformation.engine.functions.IFeatureExtraction;

import eu.dnetlib.data.collective.transformation.engine.functions.ProcessingException;

import eu.dnetlib.data.collective.transformation.engine.functions.RegularExpression;

import eu.dnetlib.data.collective.transformation.engine.functions.RetrieveValue;

import eu.dnetlib.data.collective.transformation.rulelanguage.util.FunctionCall;

import eu.dnetlib.data.collective.transformation.utils.BlacklistConsumer;

import eu.dnetlib.enabling.datasources.common.Api;

import eu.dnetlib.enabling.datasources.common.Datasource;

import eu.dnetlib.enabling.datasources.common.DsmException;

import eu.dnetlib.enabling.datasources.common.LocalDatasourceManager;

import net.sf.saxon.instruct.TerminationException;

/**

 * @author jochen

 *

 */

public class SimpleTransformationEngine {

	private LocalDatasourceManager<Datasource<?, ?, ?>, Api<?>> dsManager;

	private static Log log = LogFactory.getLog(SimpleTransformationEngine.class);

	private ITransformation transformation;

	private VocabularyRegistry vocabularyRegistry;

	private IDatabaseConnector databaseConnector;

	private ResourceDao resourceDao;

	private IFeatureExtraction featureExtraction;

	private final List<String> mdRecords = new LinkedList<>();

	private long totalTransformedRecords = 0;

	private long totalIgnoredRecords = 0;

	private String mappingFile;

	private boolean stylesheetParamsCalculated = false;

	private boolean preprocessingDone = false;

	private final Map<String, String> stylesheetParams = new LinkedHashMap<>();

	private Resource blacklistApi;

	private List<String> blacklistedRecords = new LinkedList<>();

	/**

	 * execute any preprocessings declared in the transformation script prior starting the transformation of records

	 */

	public void preprocess(final String dataSourceId) {

		for (final Map<String, String> preprocMap : this.transformation.getRuleLanguageParser().getPreprocessings()) {

			final Iterator<String> it = preprocMap.keySet().iterator();

			while (it.hasNext()) {

				final String function = it.next();

				// if (function.equals("dblookup")) {

				// Dblookup fun = new Dblookup();

				// fun.setDbConnector(databaseConnector);

				// try {

				// log.debug("preprocessingMap value: " + preprocMap.get(function));

				// TransformationFunctionProxy.getInstance().setLookupRecord(fun.getResults(preprocMap.get(function)));

				// } catch (Exception e) {

				// log.debug(e.getMessage());

				// throw new IllegalStateException(e);

				// }

				// }

				if (function.equals("blacklist")) {

					final BlacklistConsumer bc = new BlacklistConsumer();

					try {

						blacklistedRecords = bc.getBlackList(blacklistApi.getURL() + dataSourceId);

					} catch (final Exception e) {

						throw new IllegalStateException("error in preprocess: " + e.getMessage());

					}

				}

			}

		}

		log.debug("preprocessing done.");

	}

	/**

	 * check if blacklistedRecords exist and if so check if the current record is blacklisted by its objIdentifier

	 *

	 * @param aRecord

	 * @return

	 * @throws XPathExpressionException

	 * @throws ProcessingException

	 */

	private boolean isBlacklistRecord(final String aRecord) {

		if (blacklistedRecords.size() == 0) { return false; }

		final XPath xpath = XPathFactory.newInstance().newXPath();

		try {

			final Node root = (Node) xpath.evaluate("/", new InputSource(new StringReader(aRecord)), XPathConstants.NODE);

			final String objId = xpath.evaluate("//*[local-name()='objIdentifier']", root);

			if (blacklistedRecords.contains(objId)) { return true; }

		} catch (final Exception e) {

			throw new IllegalStateException("error in isBlacklistRecord: " + e.getMessage());

		}

		return false;

	}

	/**

	 * transforms a source record

	 *

	 * @param sourceRecord

	 *            the record to transform

	 * @return transformed record

	 */

	public String transform(final String sourceRecord) {

		final List<String> objectRecords = new LinkedList<>();

		objectRecords.add(sourceRecord);

		int index = 0;

		mdRecords.clear();

		initTransformationFunction();

		if (!stylesheetParamsCalculated) {

			try {

				calculateStylesheetParams(sourceRecord);

			} catch (final Exception e) {

				throw new IllegalStateException("error in calculateStylesheetParams" + e.getMessage(), e);

			}

		}

		if (!preprocessingDone) {

			// xpath sourceRecord dataSourceid

			preprocess(stylesheetParams.get("varBlacklistDataSourceId"));

			preprocessingDone = true;

		}

		if (isBlacklistRecord(sourceRecord)) {

			try {

				mdRecords.add(transformation.transformRecord(sourceRecord, ITransformation.XSLSyntaxcheckfailed));

			} catch (final Exception e) {

				log.fatal(sourceRecord);

				throw new IllegalStateException(e);

			}

		} else if (!transformation.getRuleLanguageParser().isXslStylesheet()) {

			// iterate over all rules which are functionCalls

			log.debug("functionCalls size: " + transformation.getRuleLanguageParser().getFunctionCalls().size());

			for (final FunctionCall functionCall : transformation.getRuleLanguageParser().getFunctionCalls()) {

				preprocess(objectRecords, functionCall);

			}

			for (final String record : objectRecords) {

				// log.debug(record);

				try {

					log.debug("now run transformation for record with index: " + index);

					try {

						final String transformedRecord = transformation.transformRecord(record, index);

						mdRecords.add(transformedRecord);

					} catch (final TerminationException e) {

						log.debug("record transformation terminated.");

						final String failedRecord = transformation.transformRecord(record, ITransformation.XSLSyntaxcheckfailed);

						log.debug(failedRecord);

						totalIgnoredRecords++;

						mdRecords.add(failedRecord);

					}

				} catch (final TransformationException e) {

					log.error(sourceRecord);

					throw new IllegalStateException(e);

				}

				index++;

			}

		} else {

			for (final String record : objectRecords) {

				// test for init params and assign values

				try {

					log.debug("now run transformation for record with index: " + index);

					try {

						final String transformedRecord = transformation.transformRecord(record, stylesheetParams);

						mdRecords.add(transformedRecord);

					} catch (final TerminationException e) {

						final String failedRecord = transformation.transformRecord(record, ITransformation.XSLSyntaxcheckfailed);

						totalIgnoredRecords++;

						log.debug(failedRecord);

						mdRecords.add(failedRecord);

					}

				} catch (final TransformationException e) {

					log.error(sourceRecord);

					throw new IllegalStateException(e);

				}

				index++;

			}

		}

		totalTransformedRecords = totalTransformedRecords + mdRecords.size();

		log.debug("objRecordSize: " + objectRecords.size() + ", mdRecordSize: " + mdRecords.size() + ", ignoredRecordSize: " + totalIgnoredRecords);

		return mdRecords.get(0);

	}

	private void calculateStylesheetParams(final String aRecord) throws XPathExpressionException, ProcessingException, DsmException {

		stylesheetParamsCalculated = true;

		final XPath xpath = XPathFactory.newInstance().newXPath();

		final Node root = (Node) xpath.evaluate("/", new InputSource(new StringReader(aRecord)), XPathConstants.NODE);

		final String dsId = xpath.evaluate("//*[local-name()='repositoryId']", root);

		final String nsPrefix = xpath.evaluate("//*[local-name()='datasourceprefix']", root);

		final Datasource<?, ?, ?> ds;

		if (StringUtils.isNotBlank(nsPrefix)) {

			ds = dsManager.getDsByNsPrefix(nsPrefix);

		} else if (StringUtils.isNotBlank(dsId)) { 

			ds = dsManager.getDs(dsId);

		} else {

			ds = null;

		}

		if (ds != null) {

			stylesheetParams.put("varOfficialName", ds.getOfficialname());

			stylesheetParams.put("varDataSourceId", ds.getId());

			stylesheetParams.put("varDsType", ds.getEoscDatasourceType());

		}

		// if blacklist

		for (final Map<String, String> preprocMap : this.transformation.getRuleLanguageParser().getPreprocessings()) {

			final Iterator<String> it = preprocMap.keySet().iterator();

			while (it.hasNext()) {

				final String function = it.next();

				if (function.equals("blacklist")) {

					// TODO

					// stylesheetParams.put("varBlacklistDataSourceId", varBlacklistDataSourceId);

				}

			}

		}

	}

	private void initTransformationFunction() {

		if (this.vocabularyRegistry == null) { throw new IllegalStateException("vocabularyReg is null"); }

		final Convert convertFunction = new Convert();

		convertFunction.setVocabularyRegistry(this.vocabularyRegistry);

		TransformationFunctionProxy.getInstance().setConvertFunction(convertFunction);

	}

	/**

	 * preprocesses function if function is configured resp.

	 *

	 * @param records

	 *            list of object records

	 * @param aFunctionCall

	 */

	private void preprocess(final List<String> records, final FunctionCall aFunctionCall) {

		try {

			log.debug("preprocess");

			if (transformation.getRuleLanguageParser() == null) { throw new IllegalStateException("rulelanguageparser not initialised"); }

			if (transformation.getRuleLanguageParser().getNamespaceDeclarations() == null) { throw new IllegalStateException("nsDecl is null"); }

			final PreProcessor preProc = new PreProcessor();

			preProc.setConvertFunction(TransformationFunctionProxy.getInstance().getConvertFunction());

			final RetrieveValue retrieveValue = new RetrieveValue();

			retrieveValue.setResourceDao(resourceDao);

			retrieveValue.setDsManager(dsManager);

			preProc.setRetrieveFunction(retrieveValue);

			final RegularExpression regExpr = new RegularExpression();

			preProc.setRegExprFunction(regExpr);

			final TransformationFunctionProxy functionProxy = TransformationFunctionProxy.getInstance();

			preProc.setFunctionProxy(functionProxy);

			final Extract extractFunction = new Extract();

			extractFunction.setFeatureExtraction(featureExtraction);

			preProc.setExtractFunction(extractFunction);

			if (aFunctionCall.doPreprocess() || aFunctionCall.isStatic()) {

				// log.debug("now call preprocess with: " + aFunctionCall.getExternalFunctionName() + " " + aFunctionCall.getUuid());

				preProc.preprocess(aFunctionCall, records, transformation.getRuleLanguageParser().getNamespaceDeclarations(), transformation

					.getStaticTransformationResults(), transformation.getJobProperties(), transformation.getRuleLanguageParser().getVariableMappingRules());

				// log.debug("preprocess end");

			} else {

				log.debug("skip preprocessing for function: " + aFunctionCall.getExternalFunctionName());

			}

		} catch (final Exception e) {

			throw new IllegalStateException(e);

		}

	}

	/**

	 * @param transformation

	 *            the transformation to set

	 */

	public void setTransformation(final ITransformation transformation) {

		this.transformation = transformation;

	}

	/**

	 * @return the transformation

	 */

	public ITransformation getTransformation() {

		return transformation;

	}

	/**

	 * @param vocabularyRegistry

	 *            the vocabularyRegistry to set

	 */

	public void setVocabularyRegistry(final VocabularyRegistry vocabularyRegistry) {

		this.vocabularyRegistry = vocabularyRegistry;

	}

	/**

	 * @return the vocabularyRegistry

	 */

	public VocabularyRegistry getVocabularyRegistry() {

		return vocabularyRegistry;

	}

	/**

	 * @return the resourceDao

	 */

	public ResourceDao getResourceDao() {

		return resourceDao;

	}

	/**

	 * @param resourceDao

	 *            the resourceDao to set

	 */

	public void setResourceDao(final ResourceDao resourceDao) {

		this.resourceDao = resourceDao;

	}

	/**

	 * @param featureExtraction

	 *            the featureExtraction to set

	 */

	public void setFeatureExtraction(final IFeatureExtraction featureExtraction) {

		this.featureExtraction = featureExtraction;

	}

	/**

	 * @return the featureExtraction

	 */

	public IFeatureExtraction getFeatureExtraction() {

		return featureExtraction;

	}

	/**

	 * @return the databaseConnector

	 */

	public IDatabaseConnector getDatabaseConnector() {

		return databaseConnector;

	}

	/**

	 * @param databaseConnector

	 *            the databaseConnector to set

	 */

	public void setDatabaseConnector(final IDatabaseConnector databaseConnector) {

		this.databaseConnector = databaseConnector;

	}

	public long getTotalTransformedRecords() {

		return this.totalTransformedRecords;

	}

	public long getTotalIgnoredRecords() {

		return this.totalIgnoredRecords;

	}

	/**

	 * @return the mappingFile

	 */

	public String getMappingFile() {

		return mappingFile;

	}

	/**

	 * @param mappingFile

	 *            the mappingFile to set

	 */

	public void setMappingFile(final String mappingFile) {

		this.mappingFile = mappingFile;

	}

	public Resource getBlacklistApi() {

		return blacklistApi;

	}

	public void setBlacklistApi(final Resource blacklistApi) {

		this.blacklistApi = blacklistApi;

	}

	public LocalDatasourceManager<Datasource<?, ?, ?>, Api<?>> getDsManager() {

		return dsManager;

	}

	public void setDsManager(final LocalDatasourceManager<Datasource<?, ?, ?>, Api<?>> dsManager) {

		this.dsManager = dsManager;

	}

}