D-Net

/**

 * 

 */

package eu.dnetlib.data.collective.transformation.engine.functions;

import java.util.List;

import prototype.Person;

/**

 * @author jochen

 *

 */

public class PersonVocabulary extends Vocabulary{

	@Override

	public String encoding(List<String> keys)throws ProcessingException{

		Person p;

		String result = "";

		for (String input: keys){

			p = new Person(input);

			result = p.getNormalisedFullname();

		}

		return result;

	}

}

/**

 * 

 */

package eu.dnetlib.data.collective.transformation.engine.functions;

import java.util.List;

import eu.dnetlib.data.collective.transformation.TransformationException;

/**

 * @author jochen

 *

 */

public class Extract extends AbstractTransformationFunction {

	public static final String paramNameFeature = "feature";

	private IFeatureExtraction featureExtraction;

	/* (non-Javadoc)

	 * @see eu.dnetlib.data.collective.transformation.engine.functions.AbstractTransformationFunction#execute()

	 */

	@Override

	String execute() throws ProcessingException {

		// TODO Auto-generated method stub

		return null;

	}

	public List<String> execute(List<String> aObjectRecords, String aFeature) throws ProcessingException{

		try {

			return featureExtraction.execute(aObjectRecords, aFeature);

		} catch (TransformationException e) {

			throw new ProcessingException(e);

		}

	}

	/**

	 * @param featureExtraction the featureExtraction to set

	 */

	public void setFeatureExtraction(IFeatureExtraction featureExtraction) {

		this.featureExtraction = featureExtraction;

	}

	/**

	 * @return the featureExtraction

	 */

	public IFeatureExtraction getFeatureExtraction() {

		return featureExtraction;

	}

}

/**

 * 

 */

package eu.dnetlib.data.collective.transformation.engine.functions;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.oro.text.perl.MalformedPerl5PatternException;

import org.apache.oro.text.perl.Perl5Util;

/**

 * @author jochen

 *

 */

public class RegularExpression extends AbstractTransformationFunction {

	public static final Log log = LogFactory.getLog(RegularExpression.class);

	public static final String paramRegularExpr = "regularExpression";

	public static final String paramExpr1		= "expr1";

	public static final String paramExpr2		= "expr2";

	private Perl5Util util = new Perl5Util();

	/* (non-Javadoc)

	 * @see eu.dnetlib.data.collective.transformation.engine.functions.AbstractTransformationFunction#execute()

	 */

	@Override

	String execute() throws ProcessingException {

		// TODO Auto-generated method stub

		return null;

	}

	public String executeSingleValue(String aRegularExpression, String aExpr1, String aExpr2) throws ProcessingException{

		String result = "";

		if (aRegularExpression.startsWith("m/")){

			if (util.match(aRegularExpression, aExpr1))

				result = util.group(1);

		}else if (!aRegularExpression.startsWith("s/")){ 

			// assume match and extract

			// throw new ProcessingException("unsupported or invalid regular expression: " + aRegularExpression);

			if (util.match(aRegularExpression, aExpr1)){

				String funder = util.group(1).toLowerCase();

				String projectId = util.group(3);

				result = funder + "_" + projectId;

			}

		}else{

			try{

				result = util.substitute(aRegularExpression, aExpr1);							

			}catch(MalformedPerl5PatternException patternExc){

				log.fatal("aRegularExpression: " + aRegularExpression);

				log.fatal("aExpr1: " + aExpr1);

				log.fatal(patternExc.getMessage());

				throw new ProcessingException(patternExc);

			}

		}

		return result;

	}

}

package eu.dnetlib.data.collective.transformation.engine.functions;

import java.util.List;

public abstract class AbstractTransformationFunction implements

		ITransformationFunction {

	List<String> objectRecords;

	List<String> resultRecords;

	abstract String execute() throws ProcessingException;

}

/**

 * 

 */

package eu.dnetlib.data.collective.transformation.engine.functions;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * @author jochen

 *

 */

public class Lookup extends AbstractTransformationFunction {

	public static final Log log = LogFactory.getLog(Lookup.class);

	public static final String paramExprIdentifier = "exprIdentifier";

	public static final String paramExprProperty = "exprProperty";

	/**

	 * 

	 */

	public Lookup() {

		// TODO Auto-generated constructor stub

	}

	/* (non-Javadoc)

	 * @see eu.dnetlib.data.collective.transformation.engine.functions.AbstractTransformationFunction#execute()

	 */

	@Override

	String execute() throws ProcessingException {

		// TODO Auto-generated method stub

		return null;

	}

}

/**

 * 

 */

package eu.dnetlib.data.collective.transformation.engine.functions;

import java.util.List;

/**

 * @author jochen

 *

 */

public interface IVocabulary {

	/**

	 * return the encoding for a given list of values

	 * @param keys

	 * @return the encoding as string

	 * @throws ProcessingException

	 */

	public String encoding(List<String> keys) throws ProcessingException;

	/**

	 * return the encoding for a given list of values using a default pattern and applying a filter function

	 * @param aKeys

	 * @param aDefaultPattern

	 * @param aFilterFunction

	 * @return the list of encoded values

	 * @throws ProcessingException

	 */

	public List<String> encoding(List<String> aKeys, String aDefaultPattern, String aFilterFunction) throws ProcessingException;

}

/**

 * 

 */

package eu.dnetlib.data.collective.transformation.engine.functions;

/**

 * @author jochen

 *

 */

public class ProcessingException extends Exception {

	/**

	 * 

	 */

	private static final long serialVersionUID = -8648116731979859467L;

	/**

	 * 

	 */

	public ProcessingException() {

		super();

	}

	/**

	 * @param arg0

	 */

	public ProcessingException(String arg0) {

		super(arg0);

	}

	/**

	 * @param arg0

	 */

	public ProcessingException(Throwable arg0) {

		super(arg0);

	}

	/**

	 * @param arg0

	 * @param arg1

	 */

	public ProcessingException(String arg0, Throwable arg1) {

		super(arg0, arg1);

	}

}

package eu.dnetlib.data.collective.transformation.engine;

import java.io.StringReader;

import java.util.HashMap;

import java.util.Iterator;

import java.util.LinkedHashMap;

import java.util.LinkedList;

import java.util.List;

import java.util.Map;

import javax.xml.xpath.XPath;

import javax.xml.xpath.XPathConstants;

import javax.xml.xpath.XPathExpressionException;

import javax.xml.xpath.XPathFactory;

import net.sf.saxon.instruct.TerminationException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.core.io.Resource;

import org.w3c.dom.Node;

import org.xml.sax.InputSource;

import eu.dnetlib.common.profile.ResourceDao;

import eu.dnetlib.data.collective.transformation.IDatabaseConnector;

import eu.dnetlib.data.collective.transformation.TransformationException;

import eu.dnetlib.data.collective.transformation.VocabularyRegistry;

import eu.dnetlib.data.collective.transformation.core.xsl.ext.TransformationFunctionProxy;

import eu.dnetlib.data.collective.transformation.engine.core.ITransformation;

import eu.dnetlib.data.collective.transformation.engine.functions.Convert;

// import eu.dnetlib.data.collective.transformation.engine.functions.Dblookup;

import eu.dnetlib.data.collective.transformation.engine.functions.Extract;

import eu.dnetlib.data.collective.transformation.engine.functions.IFeatureExtraction;

import eu.dnetlib.data.collective.transformation.engine.functions.ProcessingException;

import eu.dnetlib.data.collective.transformation.engine.functions.RegularExpression;

import eu.dnetlib.data.collective.transformation.engine.functions.RetrieveValue;

import eu.dnetlib.data.collective.transformation.engine.functions.RetrieveValue.FUNCTION;

import eu.dnetlib.data.collective.transformation.rulelanguage.Argument;

import eu.dnetlib.data.collective.transformation.rulelanguage.Argument.Type;

import eu.dnetlib.data.collective.transformation.rulelanguage.util.FunctionCall;

import eu.dnetlib.data.collective.transformation.utils.BlacklistConsumer;

/**

 * @author jochen

 *

 */

public class SimpleTransformationEngine {

	private static Log log = LogFactory.getLog(SimpleTransformationEngine.class);

	private ITransformation transformation;

	private VocabularyRegistry vocabularyRegistry;

	private IDatabaseConnector databaseConnector;

	private ResourceDao resourceDao;

	private IFeatureExtraction featureExtraction;

	private final List<String> mdRecords = new LinkedList<String>();

	private long totalTransformedRecords = 0;

	private long totalIgnoredRecords = 0;

	private String mappingFile;

	private boolean stylesheetParamsCalculated = false;

	private boolean preprocessingDone = false;

	private Map<String, String> stylesheetParams = new LinkedHashMap<String, String>();

	private Resource blacklistApi;

	private List<String> blacklistedRecords = new LinkedList<String>();

	/**

	 * execute any preprocessings declared in the transformation script prior starting the transformation of records

	 */

	public void preprocess(String dataSourceId) {

		for (Map<String, String> preprocMap : this.transformation.getRuleLanguageParser().getPreprocessings()) {

			Iterator<String> it = preprocMap.keySet().iterator();

			while (it.hasNext()) {

				String function = it.next();

//				if (function.equals("dblookup")) {

//					Dblookup fun = new Dblookup();

//					fun.setDbConnector(databaseConnector);

//					try {

//						log.debug("preprocessingMap value: " + preprocMap.get(function));

//						TransformationFunctionProxy.getInstance().setLookupRecord(fun.getResults(preprocMap.get(function)));

//					} catch (Exception e) {

//						log.debug(e.getMessage());

//						throw new IllegalStateException(e);

//					}

//				}

				if (function.equals("blacklist")) {

					BlacklistConsumer bc = new BlacklistConsumer();

					try{

						blacklistedRecords = bc.getBlackList(blacklistApi.getURL() + dataSourceId);						

					}catch(Exception e){

						throw new IllegalStateException("error in preprocess: " + e.getMessage());

					}

				}

			}

		}

		log.debug("preprocessing done.");

	}

	/**

	 * check if blacklistedRecords exist and if so check if the current record is blacklisted by its objIdentifier

	 * @param aRecord

	 * @return

	 * @throws XPathExpressionException

	 * @throws ProcessingException

	 */

	private boolean isBlacklistRecord(String aRecord){

		if (blacklistedRecords.size() == 0) return false;		

		XPath xpath = XPathFactory.newInstance().newXPath();

		try{

			Node root = (Node) xpath.evaluate("/", new InputSource(new StringReader(aRecord)), XPathConstants.NODE);

			String objId = xpath.evaluate("//*[local-name()='objIdentifier']", root);

			if (blacklistedRecords.contains(objId)) return true;			

		}catch(Exception e){

			throw new IllegalStateException("error in isBlacklistRecord: " + e.getMessage());

		}		

		return false;

	}

	/**

	 * transforms a source record

	 *

	 * @param sourceRecord

	 *            the record to transform

	 * @return transformed record

	 */

	public String transform(final String sourceRecord) {

		List<String> objectRecords = new LinkedList<String>();

		objectRecords.add(sourceRecord);

		int index = 0;

		mdRecords.clear();

		initTransformationFunction();

		if (!stylesheetParamsCalculated) {

			try{

				calculateStylesheetParams(sourceRecord);

			}catch(Exception e){

				throw new IllegalStateException("error in calculateStylesheetParams" + e.getMessage());

			}

		}

		if (!preprocessingDone){

			// xpath sourceRecord dataSourceid

			preprocess(stylesheetParams.get("varBlacklistDataSourceId"));

			preprocessingDone = true;

		}

		if (isBlacklistRecord(sourceRecord)){

			try{

				mdRecords.add(transformation.transformRecord(sourceRecord, ITransformation.XSLSyntaxcheckfailed));

			}catch(Exception e){

				log.fatal(sourceRecord);

				throw new IllegalStateException(e);				

			}

		}else if (!transformation.getRuleLanguageParser().isXslStylesheet()) {

			// iterate over all rules which are functionCalls

			log.debug("functionCalls size: " + transformation.getRuleLanguageParser().getFunctionCalls().size());

			for (FunctionCall functionCall : transformation.getRuleLanguageParser().getFunctionCalls()) {

				preprocess(objectRecords, functionCall);

			}

			for (String record : objectRecords) {

				// log.debug(record);

				try {

					log.debug("now run transformation for record with index: " + index);

					try{

						String transformedRecord = transformation.transformRecord(record, index);

						mdRecords.add(transformedRecord);

					} catch (TerminationException e){

						log.debug("record transformation terminated.");

						String failedRecord = transformation.transformRecord(record, ITransformation.XSLSyntaxcheckfailed);

						log.debug(failedRecord);

						totalIgnoredRecords++;

						mdRecords.add(failedRecord);

					}

				} catch (TransformationException e) {

					log.error(sourceRecord);

					throw new IllegalStateException(e);

				}

				index++;

			}

		} else {

			for (String record : objectRecords) {

				// test for init params and assign values

				try {

					log.debug("now run transformation for record with index: " + index);

					try{

						String transformedRecord = transformation.transformRecord(record, stylesheetParams);

						mdRecords.add(transformedRecord);

					}catch(TerminationException e){

						String failedRecord = transformation.transformRecord(record, ITransformation.XSLSyntaxcheckfailed);

						totalIgnoredRecords++;

						log.debug(failedRecord);

						mdRecords.add(failedRecord);

					}

				} catch (TransformationException e) {

					log.error(sourceRecord);

					throw new IllegalStateException(e);

				}

				index++;

			}

		}

		totalTransformedRecords = totalTransformedRecords + mdRecords.size();

		log.debug("objRecordSize: " + objectRecords.size() + ", mdRecordSize: " + mdRecords.size() + ", ignoredRecordSize: " + totalIgnoredRecords);

		return mdRecords.get(0);

	}

	private void calculateStylesheetParams(final String aRecord) throws XPathExpressionException, ProcessingException {

		stylesheetParamsCalculated = true;

		XPath xpath = XPathFactory.newInstance().newXPath();

		Node root = (Node) xpath.evaluate("/", new InputSource(new StringReader(aRecord)), XPathConstants.NODE);

		String datasourcePrefix = xpath.evaluate("//*[local-name()='datasourceprefix']", root);

		String profileXquery = "collection('/db/DRIVER/RepositoryServiceResources')//RESOURCE_PROFILE[.//EXTRA_FIELDS/FIELD[key=\"NamespacePrefix\"][value=\"" + datasourcePrefix + "\"]]";

		//String repositoryId = xpath.evaluate("//*[local-name()='repositoryId']", root);

		log.debug("profileXquery: " + profileXquery);

		// static $varDatasourceid = getValue(PROFILEFIELD, [xpath:"//dri:repositoryId",

		// xpath:"//EXTRA_FIELDS/FIELD[key='OpenAireDataSourceId']/value"]);

		RetrieveValue retrieveValue = new RetrieveValue();

		retrieveValue.setResourceDao(resourceDao);

		List<Argument> argList = new LinkedList<Argument>();

		argList.add(new Argument(Type.VALUE, profileXquery));

		Argument argXpath = new Argument(Type.INPUTFIELD, "//OFFICIAL_NAME");

		argList.add(argXpath);

		String varOfficialName = retrieveValue.executeSingleValue(FUNCTION.PROFILEFIELD.toString(), argList, null, new HashMap<String, String>());

		stylesheetParams.put("varOfficialName", varOfficialName);

		argList.remove(argXpath);

		argXpath = new Argument(Type.INPUTFIELD, "//EXTRA_FIELDS/FIELD[key='OpenAireDataSourceId']/value");

		argList.add(argXpath);

		String varDataSourceId = retrieveValue.executeSingleValue(FUNCTION.PROFILEFIELD.toString(), argList, null, new HashMap<String, String>());

		stylesheetParams.put("varDataSourceId", varDataSourceId);

		argList.remove(argXpath);

		argXpath = new Argument(Type.INPUTFIELD, "//CONFIGURATION/DATASOURCE_TYPE");

		argList.add(argXpath);

		String varDsType = retrieveValue.executeSingleValue(FUNCTION.PROFILEFIELD.toString(), argList, null, new HashMap<String, String>());

		stylesheetParams.put("varDsType", varDsType);

		argList.remove(argXpath);

		// if blacklist

		for (Map<String, String> preprocMap : this.transformation.getRuleLanguageParser().getPreprocessings()) {

			Iterator<String> it = preprocMap.keySet().iterator();

			while (it.hasNext()) {	

				String function = it.next();

				if (function.equals("blacklist")) {

					argXpath = new Argument(Type.INPUTFIELD, preprocMap.get(function)); // blacklistDataSourceIdXpath

					argList.add(argXpath);

					String varBlacklistDataSourceId = retrieveValue.executeSingleValue(FUNCTION.PROFILEFIELD.toString(), argList, null, new HashMap<String, String>());

					stylesheetParams.put("varBlacklistDataSourceId", varBlacklistDataSourceId);					

					argList.remove(argXpath);

				}

			}

		}

	}

	private void initTransformationFunction() {

		if (this.vocabularyRegistry == null) { throw new IllegalStateException("vocabularyReg is null"); }

		Convert convertFunction = new Convert();

		convertFunction.setVocabularyRegistry(this.vocabularyRegistry);

		TransformationFunctionProxy.getInstance().setConvertFunction(convertFunction);

	}

	/**

	 * preprocesses function if function is configured resp.

	 *

	 * @param records

	 *            list of object records

	 * @param aFunctionCall

	 */

	private void preprocess(final List<String> records, final FunctionCall aFunctionCall) {

		try {

			log.debug("preprocess");

			if (transformation.getRuleLanguageParser() == null) { throw new IllegalStateException("rulelanguageparser not initialised"); }

			if (transformation.getRuleLanguageParser().getNamespaceDeclarations() == null) { throw new IllegalStateException("nsDecl is null"); }

			PreProcessor preProc = new PreProcessor();

			preProc.setConvertFunction(TransformationFunctionProxy.getInstance().getConvertFunction());

			RetrieveValue retrieveValue = new RetrieveValue();

			retrieveValue.setResourceDao(resourceDao);

			preProc.setRetrieveFunction(retrieveValue);

			RegularExpression regExpr = new RegularExpression();

			preProc.setRegExprFunction(regExpr);

			TransformationFunctionProxy functionProxy = TransformationFunctionProxy.getInstance();

			preProc.setFunctionProxy(functionProxy);

			Extract extractFunction = new Extract();

			extractFunction.setFeatureExtraction(featureExtraction);

			preProc.setExtractFunction(extractFunction);

			if (aFunctionCall.doPreprocess() || aFunctionCall.isStatic()) {

				// log.debug("now call preprocess with: " + aFunctionCall.getExternalFunctionName() + " " + aFunctionCall.getUuid());

				preProc.preprocess(

						aFunctionCall,

						records,

						transformation.getRuleLanguageParser().getNamespaceDeclarations(),

						transformation.getStaticTransformationResults(),

						transformation.getJobProperties(),

						transformation.getRuleLanguageParser().getVariableMappingRules());

				// log.debug("preprocess end");

			} else {

				log.debug("skip preprocessing for function: " + aFunctionCall.getExternalFunctionName());

			}

		} catch (Exception e) {

			throw new IllegalStateException(e);

		}

	}

	/**

	 * @param transformation

	 *            the transformation to set

	 */

	public void setTransformation(final ITransformation transformation) {

		this.transformation = transformation;

	}

	/**

	 * @return the transformation

	 */

	public ITransformation getTransformation() {

		return transformation;

	}

	/**

	 * @param vocabularyRegistry

	 *            the vocabularyRegistry to set

	 */

	public void setVocabularyRegistry(final VocabularyRegistry vocabularyRegistry) {

		this.vocabularyRegistry = vocabularyRegistry;

	}

	/**

	 * @return the vocabularyRegistry

	 */

	public VocabularyRegistry getVocabularyRegistry() {

		return vocabularyRegistry;

	}

	/**

	 * @return the resourceDao

	 */

	public ResourceDao getResourceDao() {

		return resourceDao;

	}

	/**

	 * @param resourceDao

	 *            the resourceDao to set

	 */

	public void setResourceDao(final ResourceDao resourceDao) {

		this.resourceDao = resourceDao;

	}

	/**

	 * @param featureExtraction

	 *            the featureExtraction to set

	 */

	public void setFeatureExtraction(final IFeatureExtraction featureExtraction) {

		this.featureExtraction = featureExtraction;

	}

	/**

	 * @return the featureExtraction

	 */

	public IFeatureExtraction getFeatureExtraction() {

		return featureExtraction;

	}

	/**

	 * @return the databaseConnector

	 */

	public IDatabaseConnector getDatabaseConnector() {

		return databaseConnector;

	}

	/**

	 * @param databaseConnector

	 *            the databaseConnector to set

	 */

	public void setDatabaseConnector(final IDatabaseConnector databaseConnector) {

		this.databaseConnector = databaseConnector;

	}

	public long getTotalTransformedRecords() {

		return this.totalTransformedRecords;

	}

	public long getTotalIgnoredRecords() {

		return this.totalIgnoredRecords;

	}

	/**

	 * @return the mappingFile

	 */

	public String getMappingFile() {

		return mappingFile;

	}

	/**

	 * @param mappingFile

	 *            the mappingFile to set

	 */

	public void setMappingFile(final String mappingFile) {

		this.mappingFile = mappingFile;

	}

	public Resource getBlacklistApi() {

		return blacklistApi;

	}

	public void setBlacklistApi(Resource blacklistApi) {

		this.blacklistApi = blacklistApi;

	}

}

package eu.dnetlib.data.collective.transformation.engine;

import java.util.Collection;

import java.util.LinkedHashMap;

import java.util.LinkedList;

import java.util.List;

import java.util.Map;

/**

 * @author js

 *

 */

public class FunctionResults {

	private Map<String, List<String>> resultMap = new LinkedHashMap<String, List<String>>();

	/**

	 * get the first single result from the result list at the given index

	 * @param aIndex

	 * @return a result

	 */

	public String get(int aIndex){

		return resultMap.get(aIndex + "").get(0);

	}

	/**

	 * get the single result for the node at the given position from the list at the given index

	 * @param aIndex

	 * @param aPosition

	 * @return a result

	 */

	public String get(int aIndex, int aPosition){

		if (aPosition <= 0){

			throw new IllegalArgumentException("position is " + aPosition + ", must be greater 0");

		}

		return resultMap.get(aIndex + "").get(aPosition - 1);

	}

	/**

	 * add a collection containing the results for each record

	 * @param aCollection

	 */

	public void addAll(Collection<? extends String> aCollection){

		for (String result : aCollection){

			add(result);

		}

	}

	/**

	 * add a single result calculated for a record node

	 * @param aResult

	 */

	public void add(String aResult){

		List<String> resultList = new LinkedList<String>();

		resultList.add(aResult);

		resultMap.put(resultMap.size() + "", resultList);

	}

	/**

	 * add a list of results calculated for all resp. record nodes

	 * @param aResults

	 */

	public void add(List<String> aResults){

		resultMap.put(resultMap.size() + "", aResults);

	}

}

package eu.dnetlib.data.collective.transformation.utils;

import java.util.HashMap;

import java.util.Iterator;

import java.util.Map;

import javax.xml.XMLConstants;

import javax.xml.namespace.NamespaceContext;

public class NamespaceContextImpl implements NamespaceContext {

	private Map<String, String> nsMap = new HashMap<String, String>();

	public void addNamespace(String aPrefix, String aURI){

		nsMap.put(aPrefix, aURI);

	}

	@Override

	public String getNamespaceURI(String aPrefix) {

		return nsMap.get(aPrefix);

	}

	@Override

	public String getPrefix(String aNamespaceURI) {

		if (aNamespaceURI == null){

			throw new IllegalStateException();

		}

		if (aNamespaceURI.equals(XMLConstants.XML_NS_URI)){

			return XMLConstants.XML_NS_PREFIX;

		}else if (aNamespaceURI.equals(XMLConstants.XMLNS_ATTRIBUTE_NS_URI)){

			return XMLConstants.XMLNS_ATTRIBUTE;

		}else if (nsMap.values().contains(aNamespaceURI)){

			for (String prefix: nsMap.keySet()){

				if (nsMap.get(prefix).equals(aNamespaceURI)){

					return prefix;

				}

			}

		}

		return null;

	}

	@Override

	public Iterator getPrefixes(String arg0) {

		// TODO Auto-generated method stub

		return null;

	}

}

package eu.dnetlib.data.collective.transformation.utils;

import java.io.StringReader;

import java.util.List;

import org.apache.commons.text.StringEscapeUtils;

import eu.dnetlib.common.profile.ProfileNotFoundException;

import eu.dnetlib.data.collective.transformation.rulelanguage.RuleLanguageParser;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.tools.ServiceLocator;

/**

 * 

 * @author jochen

 * @since 1.2

 */

public class TransformationRulesImportTool {

	private ServiceLocator<ISLookUpService> lookupServiceLocator;

	/**

	 * retrieves the transformation rule script of a transformation rule profile identified by a profile id

	 * @param aProfileId

	 * @return list of the transformation rule script and optionally profile id's of subscripts

	 * @throws ProfileNotFoundException

	 */

	protected List<String> getScript(String aProfileId) throws ProfileNotFoundException{

 		String xquery =  "collection('/db/DRIVER/TransformationRuleDSResources')//RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value ='" +

 			aProfileId + "']//CODE/child::node(), " +

 			"for $id in (collection('/db/DRIVER/TransformationRuleDSResources')//RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value ='" +

 			aProfileId + "']//IMPORTED/SCRIPT_REFERENCE/@id) return string($id)";

		List<String> queryResult;

		try {

			queryResult = lookupServiceLocator.getService().quickSearchProfile(xquery);

			if (!queryResult.isEmpty()){

				return queryResult;

			}else{

				throw new ProfileNotFoundException("no script found in profile for profileId: " + aProfileId);

			}

		} catch (ISLookUpException e) {

			throw new ProfileNotFoundException(e);

		}

	}

	protected void importRules(RuleLanguageParser aParser, String aProfileId) throws ProfileNotFoundException{

		List<String> profileQueryResult = getScript(aProfileId);

		String script = StringEscapeUtils.unescapeXml(profileQueryResult.get(0)); // the first entry contains the script

		if (script.trim().startsWith("<xsl:stylesheet")){

			aParser.setXslStylesheet(script.trim());

		}else{

			StringReader reader = new StringReader(script);

			aParser.parse(reader);

			if (aParser.getImportedScripts().size() != (profileQueryResult.size() - 1) ) throw new IllegalStateException(

					"invalid number of scripts to import: " + aParser.getImportedScripts().size() + " != " + profileQueryResult.size());

			// recursiv

			if (!aParser.getImportedScripts().isEmpty()){

				for (String importScriptProfileId: profileQueryResult.subList(1, profileQueryResult.size())){

					RuleLanguageParser childParser = new RuleLanguageParser();

					importRules(childParser, importScriptProfileId);

					aParser.addRulesFromParser(childParser);

				}

			}

		}

	}

	/**

	 * gets the rule language parser, which creates the mapping of rules which is defined in a transformation rule script identified by the transformation rule profile id

	 * @param aProfileId - id of the transformation rules profile

	 * @return the rule language parser

	 * @throws ProfileNotFoundException

	 */

	public RuleLanguageParser getRuleLanguageParser(String aProfileId) throws ProfileNotFoundException{

		RuleLanguageParser parser = new RuleLanguageParser();

		importRules(parser, aProfileId);

		return parser;

	}

	public ServiceLocator<ISLookUpService> getLookupServiceLocator() {

		return lookupServiceLocator;

	}

	public void setLookupServiceLocator(

			ServiceLocator<ISLookUpService> lookupServiceLocator) {

		this.lookupServiceLocator = lookupServiceLocator;

	}

}

package eu.dnetlib.data.collective.transformation.core.schema.visitor;

import java.util.Collection;

import java.util.Iterator;

import org.apache.commons.lang3.NotImplementedException;

import com.sun.xml.xsom.XSAttributeUse;

import com.sun.xml.xsom.XSContentType;

import com.sun.xml.xsom.XSElementDecl;

import com.sun.xml.xsom.XSParticle;

import com.sun.xml.xsom.XSSimpleType;

import com.sun.xml.xsom.XSType;

import com.sun.xml.xsom.visitor.XSContentTypeVisitor;

import eu.dnetlib.data.collective.transformation.core.schema.SchemaAttribute;

import eu.dnetlib.data.collective.transformation.core.schema.SchemaElement;

/**

 * @author jochen

 *

 */

public class XSContentTypeVisitorImpl implements XSContentTypeVisitor {

	private Visitor visitor;

	@Override

	public void empty(XSContentType arg0) {

		throw new NotImplementedException("TODO empty");

	}

	@Override

	public void particle(XSParticle aParticle) {

		XSTermVisitorImpl termVisitor = new XSTermVisitorImpl();

		termVisitor.setVisitor(this.visitor);

		aParticle.getTerm().visit(termVisitor);		

		if (aParticle.getTerm().isElementDecl()){

			XSElementDecl elem = aParticle.getTerm().asElementDecl();						

			SchemaElement element = new SchemaElement();

			XSType type = elem.getType();

			if (type.isComplexType()){

				Collection<? extends XSAttributeUse> attrColls = 

						type.asComplexType().getDeclaredAttributeUses();

				Iterator<? extends XSAttributeUse> attrIter = attrColls.iterator();

				while (attrIter.hasNext()){

					XSAttributeUse attrUse = attrIter.next();

					SchemaAttribute attribute = new SchemaAttribute();

					attribute.setName(attrUse.getDecl().getName());

					attribute.setRequired(attrUse.isRequired());

					element.addAttribute(attribute);					

				}

			}

			element.setName(elem.getName());

			element.setTargetNamespace(elem.getTargetNamespace());

			element.setMinOccurs(aParticle.getMinOccurs().intValue());

			element.setMaxOccurs(aParticle.getMaxOccurs().intValue());

			element.setRepeatable(aParticle.isRepeated());

			if (elem.getType().isComplexType()){

				if (elem.getType().asComplexType().getContentType().asSimpleType() != null){

					element.setContainsSimpleType(true);

				}

			}			

			this.visitor.getCurrentElement().getChildList().add(element);

		}

	}

	@Override

	public void simpleType(XSSimpleType arg0) {

		throw new NotImplementedException("TODO simpleType");

	}

	public void setVisitor(Visitor visitor) {

		this.visitor = visitor;

	}

	public Visitor getVisitor() {

		return visitor;

	}

}

package eu.dnetlib.data.collective.transformation.core.schema.visitor;

import java.util.LinkedList;

import java.util.List;

import org.apache.commons.lang3.NotImplementedException;

//import org.apache.commons.logging.Log;

//import org.apache.commons.logging.LogFactory;

import com.sun.xml.xsom.XSAnnotation;

import com.sun.xml.xsom.XSAttGroupDecl;

import com.sun.xml.xsom.XSAttributeDecl;

import com.sun.xml.xsom.XSAttributeUse;

import com.sun.xml.xsom.XSComplexType;

import com.sun.xml.xsom.XSContentType;

import com.sun.xml.xsom.XSElementDecl;

import com.sun.xml.xsom.XSFacet;

import com.sun.xml.xsom.XSIdentityConstraint;

import com.sun.xml.xsom.XSModelGroup;

import com.sun.xml.xsom.XSModelGroupDecl;

import com.sun.xml.xsom.XSNotation;

import com.sun.xml.xsom.XSParticle;

import com.sun.xml.xsom.XSSchema;

import com.sun.xml.xsom.XSSimpleType;

import com.sun.xml.xsom.XSType;

import com.sun.xml.xsom.XSWildcard;

import com.sun.xml.xsom.XSXPath;

import com.sun.xml.xsom.visitor.XSVisitor;

import eu.dnetlib.data.collective.transformation.core.schema.SchemaAttribute;

import eu.dnetlib.data.collective.transformation.core.schema.SchemaElement;

/**

 * @author jochen

 *

 */

public class Visitor implements XSVisitor {

	//private static Log log = LogFactory.getLog(Visitor.class);

	List<SchemaElement> schemaElements = new LinkedList<SchemaElement>();

	SchemaElement currentElement;

	SchemaAttribute currentAttribute;

	@Override

	public void annotation(XSAnnotation arg0) {

		throw new NotImplementedException("TODO: annotation");

	}

	@Override

	public void attGroupDecl(XSAttGroupDecl arg0) {

		throw new NotImplementedException("TODO attGroupDecl");

	}

	@Override

	public void attributeDecl(XSAttributeDecl aAttributeDecl) {

		currentAttribute.setName(aAttributeDecl.getName());

		//log.debug("visit attribute name: " + aAttributeDecl.getName());

		//log.debug("visit attribute type: " + aAttributeDecl.getType());

		throw new NotImplementedException("TODO attributeDecl");

	}

	@Override

	public void attributeUse(XSAttributeUse aAttributeUse) {

		throw new NotImplementedException("TODO attributeUse");

	}

	@Override

	public void complexType(XSComplexType aType) {

		if (aType.getDerivationMethod()== XSType.RESTRICTION){

			XSContentTypeVisitorImpl contentTypeVisitor = new XSContentTypeVisitorImpl();

			contentTypeVisitor.setVisitor(this);

			aType.getContentType().visit(contentTypeVisitor);				

		}else{

			// aType.getExplicitContent().visit(this);

			throw new NotImplementedException("visiting types other then 'RESTRICTION are not implemented'");

		}

	}

	@Override

	public void facet(XSFacet arg0) {

		throw new NotImplementedException("TODO facet");

	}

	@Override

	public void identityConstraint(XSIdentityConstraint arg0) {

		throw new NotImplementedException("TODO identityConstraint");

	}

	@Override

	public void notation(XSNotation arg0) {

		throw new NotImplementedException("TODO notation");

	}

	@Override

	public void schema(XSSchema arg0) {

		throw new NotImplementedException("TODO schema");

	}

	@Override

	public void xpath(XSXPath arg0) {

		throw new NotImplementedException("TODO xpath");

	}

	@Override

	public void elementDecl(XSElementDecl aElementDecl) {

		XSType type = aElementDecl.getType();

		if (type.isLocal()){

			// complete infos about the current element

			// log.debug("visitor element name: " + aElementDecl.getName());

			currentElement.setName(aElementDecl.getName());

			currentElement.setTargetNamespace(aElementDecl.getTargetNamespace());

			type.visit(this);

		}

	}

	@Override

	public void modelGroup(XSModelGroup aGroup) {

		// a group of elements as childs of the root element

		for (XSParticle p: aGroup.getChildren()){

			particle(p);

		}

	}

	@Override

	public void modelGroupDecl(XSModelGroupDecl arg0) {

		throw new NotImplementedException("TODO modelGroupDecl");

	}

	@Override

	public void wildcard(XSWildcard arg0) {

		throw new NotImplementedException("TODO wildcard");

	}

	@Override

	public void empty(XSContentType arg0) {

		throw new NotImplementedException("TODO empty");

	}

	@Override

	public void particle(XSParticle aParticle) {

		// create a new schema element, add to the list of schema elements, set this element as current element

		SchemaElement element = new SchemaElement();

		element.setMinOccurs(aParticle.getMinOccurs().intValue());

		element.setMaxOccurs(aParticle.getMaxOccurs().intValue());

		element.setRepeatable(aParticle.isRepeated());

		schemaElements.add(element);

		currentElement = element;

		XSTermVisitorImpl termVisitor = new XSTermVisitorImpl();

		termVisitor.setVisitor(this);

		aParticle.getTerm().visit(termVisitor);

	}

	@Override

	public void simpleType(XSSimpleType arg0) {

		throw new NotImplementedException("TODO simpleType");

	}

	public List<SchemaElement> getElements(){

		return this.schemaElements;

	}

	protected SchemaElement getCurrentElement(){

		return currentElement;

	}

}

/**

 * 

 */

package eu.dnetlib.data.collective.transformation.core.schema;

/**

 * @author jochen

 *

 */

public class Namespace {

	String prefix;

	String uri;

	public Namespace(String aPrefix, String aUri) {

		this.prefix = aPrefix;

		this.uri = aUri;

	}

	/**

	 * @return the prefix

	 */

	public String getPrefix() {

		return prefix;

	}

	/**

	 * @param prefix the prefix to set

	 */

	public void setPrefix(String prefix) {

		this.prefix = prefix;

	}

	/**

	 * @return the uri

	 */

	public String getUri() {

		return uri;

	}

	/**

	 * @param uri the uri to set

	 */

	public void setUri(String uri) {

		this.uri = uri;

	}

}

package eu.dnetlib.data.collective.transformation.core.xsl;

import java.util.LinkedList;

import java.util.List;

/**

 * @author jochen

 *

 */

public abstract class AbstractXslElement {

	private String functionName;

	protected List<String> attrList = new LinkedList<String>();

	protected StringBuilder enclosedElements = new StringBuilder();

	protected List<String> nsList = new LinkedList<String>();

	public AbstractXslElement(String aFunctioName) {

		this.functionName = aFunctioName;

	}

	public String asXml(boolean isEmpty){

		StringBuilder builder = new StringBuilder();

		builder.append("<");

		builder.append(functionName + " ");

		for (String ns: nsList){

			builder.append(ns + " ");			

		}

		for (String attr: attrList){

			builder.append(attr);

		}

		if (isEmpty){

			builder.append("/>");

		}else{

			builder.append(">");

			builder.append(enclosedElements.toString());

			builder.append("</");

			builder.append(functionName + ">");			

		}

		return builder.toString();		

	}

	public String asXml() {

		return asXml(false);

	}

}

/**

 * 

 */

package eu.dnetlib.data.collective.transformation.core.schema;

/**

 * @author jochen

 *

 */

public class SchemaAttribute {

	private String name;

	private boolean required;

	/**

	 * @return the name

	 */

	public String getName() {

		return name;

	}

	/**

	 * @param name the name to set

	 */

	public void setName(String name) {

		this.name = name;

	}

	/**

	 * @return the required

	 */

	public boolean isRequired() {

		return required;

	}

	/**

	 * @param required the required to set

	 */

	public void setRequired(boolean required) {

		this.required = required;

	}

}

<?xml version="1.0" ?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>unibi-data-collective-transformation-common</artifactId>

	<packaging>jar</packaging>

	<version>2.2.13-SNAPSHOT</version>

	<scm>

	  <developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/unibi-data-collective-transformation-common/trunk</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>commons-beanutils</groupId>

			<artifactId>commons-beanutils</artifactId>

			<version>1.8.3</version>

		</dependency>

		<dependency>

			<groupId>org.apache.commons</groupId>

			<artifactId>commons-text</artifactId>

			<version>1.3</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.mockito</groupId>

			<artifactId>mockito-core</artifactId>

			<version>${mockito.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.svenson</groupId>

			<artifactId>svenson-json</artifactId>

			<version>[1.4.0,1.5.0)</version>

		</dependency>

		<dependency>

			<groupId>org.slf4j</groupId>

			<artifactId>slf4j-api</artifactId>

			<version>1.6.4</version>

			<scope>compile</scope>

		</dependency>

		<dependency>

			<groupId>org.slf4j</groupId>

			<artifactId>slf4j-log4j12</artifactId>

			<version>1.6.4</version>

			<scope>runtime</scope>

		</dependency>

		<dependency>

			<groupId>com.google.guava</groupId>

			<artifactId>guava</artifactId>

			<version>${google.guava.version}</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-service-common</artifactId>

			<version>[2.1.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>com.sun.xsom</groupId>

			<artifactId>xsom</artifactId>

			<version>20110809</version>

		</dependency>		

	</dependencies>

</project>

package eu.dnetlib.data.collective.transformation.core.xsl;

/**

 * 

 * @author js

 *

 */

public class XsltConstants {

	public static final String applyTemplates = "xsl:apply-templates";

	public static final String attribute    = "xsl:attribute";

	public static final String callTemplate = "xsl:call-template";

	public static final String choose       = "xsl:choose";

	public static final String copy         = "xsl:copy";

	public static final String copyOf       = "xsl:copy-of";

	public static final String element      = "xsl:element";

	public static final String forEach 	    = "xsl:for-each";

	public static final String ifCondition  = "xsl:if";

	public static final String message		= "xsl:message";

	public static final String otherwise    = "xsl:otherwise";

	public static final String param        = "xsl:param";

	public static final String template	    = "xsl:template";

	public static final String text         = "xsl:text";

	public static final String valueOf 	    = "xsl:value-of";

	public static final String variable	    = "xsl:variable";

	public static final String when         = "xsl:when";

	public static final String withParam    = "xsl:with-param";

	public static final String nsXsl	    = "xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\"";

	public static final String extFuncNS    = "TransformationFunction";

}

package eu.dnetlib.data.collective.transformation.core.schema;

import java.io.File;

import java.io.IOException;

import java.net.URL;

import java.util.List;

import org.xml.sax.SAXException;

import com.sun.xml.xsom.XSContentType;

import com.sun.xml.xsom.XSElementDecl;

import com.sun.xml.xsom.XSParticle;

import com.sun.xml.xsom.XSSchemaSet;

import com.sun.xml.xsom.XSTerm;

import com.sun.xml.xsom.parser.XSOMParser;

import eu.dnetlib.data.collective.transformation.core.schema.visitor.Visitor;

/**

 * @author jochen

 *

 */

public class SchemaInspector {

	private List<SchemaElement> elementList = new java.util.LinkedList<SchemaElement>();

	private boolean inspected = false;

	private String rootElement;

	public void inspect(File aSchema, String aRootElement) throws SAXException, IOException{

		XSOMParser parser = new  XSOMParser();

		parser.parse(aSchema);		

		doInspect(parser, aRootElement);

	}

	public void inspect(URL aSchema, String aRootElement)throws SAXException{

		XSOMParser parser = new  XSOMParser();

		parser.parse(aSchema);

		doInspect(parser, aRootElement);

	}

	/**

	 * inspects the schema and creates a new list of schema elements.

	 * @param parser