D-Net

<?xml version="1.0" ?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>unibi-data-collective-transformation-common</artifactId>

	<packaging>jar</packaging>

	<version>2.2.7</version>

	<scm>

	  <developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/unibi-data-collective-transformation-common/tags/unibi-data-collective-transformation-common-2.2.7</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>commons-beanutils</groupId>

			<artifactId>commons-beanutils</artifactId>

			<version>1.8.3</version>

		</dependency>

		<dependency>

			<groupId>org.apache.commons</groupId>

			<artifactId>commons-text</artifactId>

			<version>1.3</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.mockito</groupId>

			<artifactId>mockito-core</artifactId>

			<version>${mockito.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.svenson</groupId>

			<artifactId>svenson-json</artifactId>

			<version>[1.4.0,1.5.0)</version>

		</dependency>

		<dependency>

			<groupId>org.slf4j</groupId>

			<artifactId>slf4j-api</artifactId>

			<version>1.6.4</version>

			<scope>compile</scope>

		</dependency>

		<dependency>

			<groupId>org.slf4j</groupId>

			<artifactId>slf4j-log4j12</artifactId>

			<version>1.6.4</version>

			<scope>runtime</scope>

		</dependency>

		<dependency>

			<groupId>com.google.guava</groupId>

			<artifactId>guava</artifactId>

			<version>${google.guava.version}</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-service-common</artifactId>

			<version>[2.1.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>com.sun.xsom</groupId>

			<artifactId>xsom</artifactId>

			<version>20110809</version>

		</dependency>		

	</dependencies>

</project>

/**

 * 

 */

package eu.dnetlib.data.collective.transformation.engine.functions;

import java.util.HashMap;

import java.util.LinkedHashMap;

import java.util.Properties;

/**

 * @author jochen

 *

 */

public class LookupRecord {

	private HashMap<String, Properties> recordMap = new LinkedHashMap<String, Properties>();

	public void setRecord(String aRecordKey, String aPropertyKey, String aPropertyValue){

		if (recordMap.containsKey(aRecordKey)){

			recordMap.get(aRecordKey).setProperty(aPropertyKey, aPropertyValue);

		}else{

			Properties p = new Properties();

			p.setProperty(aPropertyKey, aPropertyValue);

			recordMap.put(aRecordKey, p);

		}

	}

	public String getPropertyValue(String aRecordKey, String aPropertyKey){

		if (!recordMap.containsKey(aRecordKey))	return "UNKNOWN";

		return recordMap.get(aRecordKey).getProperty(aPropertyKey, "UNKNOWN");

	}

}

/**

 * 

 */

package eu.dnetlib.data.collective.transformation.engine.functions;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.Arrays;

import java.util.Date;

//import java.util.HashMap;

import java.util.LinkedList;

import java.util.List;

//import java.util.Map;

import org.apache.oro.text.perl.Perl5Util;

/**

 * @author jochen

 *

 */

public class DateVocabulary extends Vocabulary{

	private static final String filterFuncMin = "min()";

	private String pattern_1 = "/^(\\d{4,4}-\\d{1,2}-\\d{1,2})/";

	private String pattern_2 = "/^(\\d{4,4}-\\d{1,2})$/";

	private String pattern_3 = "/^(\\d{4,4})$/";

	private String pattern_4 = "/^(\\d{1,2}.\\d{1,2}.\\d{4,4})$/";

	private SimpleDateFormat df;

	private transient Perl5Util perl5 = new Perl5Util();

	public String encoding(List<String> aKeys) throws ProcessingException{

		String tempKey_1 = null;

		String tempKey_2 = null;

		String tempKey_3 = null;

		String currentKey = null;

		String twoDigitFormat = String.format("%%0%dd", 2);

		try{

			for (String key: aKeys){

				key = key.trim();

				currentKey = key;

				if (perl5.match(pattern_1, key)){

					String[] dateSplitted = perl5.getMatch().toString().split("-");

					String dateNormalized = dateSplitted[0] + "-" + String.format(twoDigitFormat, Integer.parseInt(dateSplitted[1])) + "-" + String.format(twoDigitFormat, Integer.parseInt(dateSplitted[2]));

					return dateNormalized;

				}else if (perl5.match(pattern_2, key)){

					String[] dateSplitted = key.split("-");

					tempKey_1 = dateSplitted[0] + "-" + String.format(twoDigitFormat, Integer.parseInt(dateSplitted[1])) + "-01";

				}else if (perl5.match(pattern_3, key)){

					tempKey_2 = key + "-01-01";

				}else if (perl5.match(pattern_4, key)){

					String[] components = key.split("[\\-\\/\\.]");

					// ignore this key if it has less than 3 components

					if (components.length >= 3)

						tempKey_3 = components[2] + "-" + String.format(twoDigitFormat, Integer.parseInt(components[1])) + "-" + String.format(twoDigitFormat, Integer.parseInt(components[0]));					

				}

			}			

		}catch(Throwable e){

			throw new ProcessingException("Exception thrown in Datevocabulary (tried to match for value '" + currentKey + "'):", e);

		}

		if (tempKey_1 != null){

			return tempKey_1;

		}else if (tempKey_2 != null){

			return tempKey_2;

		}else if (tempKey_3 != null){

			return tempKey_3;

		}else{

			return "";			

		}

	}

	@Override

	public List<String> encoding(List<String> aKeys, String aDefaultPattern,

			String aFilterFunction) throws ProcessingException {

		List<String> evList = new LinkedList<String>();

		df = new SimpleDateFormat(aDefaultPattern);

		for (String v: aKeys){

			String ev = encoding(Arrays.asList(new String[]{v}));

			if (ev.length() > 0){

				try {

					if (aFilterFunction.trim().length() > 0 && !evList.isEmpty())

						evList.add( filter(df.parse(ev), df.parse(evList.remove(0)), aFilterFunction) );

					else

						evList.add(df.format(df.parse(ev)));

				} catch (ParseException e) {

					throw new ProcessingException("invalid date format: " + ev);

				}

			}

		}

		return evList;

	}

	private String filter(Date d1, Date d2, String filter) throws ProcessingException{

		if (filter.equals(filterFuncMin))

			if (d1.before(d2))

				return df.format(d1);

			else

				return df.format(d2);

		else

			throw new ProcessingException("unsupported filter function: " + filter);

	}

}

/**

 * 

 */

package eu.dnetlib.data.collective.transformation.engine.functions;

import java.util.List;

/**

 * @author jochen

 *

 */

public interface IVocabulary {

	/**

	 * return the encoding for a given list of values

	 * @param keys

	 * @return the encoding as string

	 * @throws ProcessingException

	 */

	public String encoding(List<String> keys) throws ProcessingException;

	/**

	 * return the encoding for a given list of values using a default pattern and applying a filter function

	 * @param aKeys

	 * @param aDefaultPattern

	 * @param aFilterFunction

	 * @return the list of encoded values

	 * @throws ProcessingException

	 */

	public List<String> encoding(List<String> aKeys, String aDefaultPattern, String aFilterFunction) throws ProcessingException;

}

package eu.dnetlib.data.collective.transformation.engine.functions;

import java.util.HashSet;

import java.util.LinkedList;

import java.util.List;

import java.util.Set;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

import javax.xml.parsers.ParserConfigurationException;

import javax.xml.xpath.XPath;

import javax.xml.xpath.XPathConstants;

import javax.xml.xpath.XPathExpressionException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.w3c.dom.Document;

import org.w3c.dom.DocumentFragment;

import org.w3c.dom.Element;

import org.w3c.dom.Node;

import org.w3c.dom.NodeList;

public class IdentifierExtract extends AbstractTransformationFunction{

	public static final Log log = LogFactory.getLog(IdentifierExtract.class);

	public static final String paramXpathExprJson = "xpathExprJson";

	public static final String paramXpathExprInSource = "xpathExprInputSource";

	public static final String paramRegExpr = "regExpr";

	@Override

	String execute() throws ProcessingException {

		// TODO Auto-generated method stub

		return null;

	}

	/**

	 * extract content matched by a regular expression pattern from a given node and return matched content as a node-list

	 * @param aXpathExprList

	 * @param aInput

	 * @param aRegExpression

	 * @param aDocument

	 * @param aXpath

	 * @return nodeList

	 * @throws ProcessingException

	 */

	public NodeList extract(List<String> aXpathExprList, Node aInput,

			String aRegExpression, Document aDocument, XPath aXpath) throws ProcessingException {

		log.debug("xpathExprList: " + aXpathExprList);

		log.debug("regExpr: " + aRegExpression);

		Set<String> identifierSet = new HashSet<String>();

//		log.debug("node: length: " + aInput.getChildNodes().getLength());

		log.debug("regular expression : " + aRegExpression);

		Pattern p = Pattern.compile(aRegExpression);

		try {

			List<String> textList = extractText(aXpathExprList, aInput, aXpath);

			for (String text: textList){

				log.debug("text as input : " + text);

				Matcher m = p.matcher(text);

				while (m.find()){

					log.debug("extracted identifier: " + m.group());

					identifierSet.add(m.group());

				}

			}

			return toNodeList(identifierSet, aDocument);

		} catch (XPathExpressionException e) {

			e.printStackTrace();

			throw new ProcessingException(e);

		} catch (ParserConfigurationException e) {

			e.printStackTrace();

			throw new ProcessingException(e);

		}

	}

	/**

	 * create a list of nodes from a list of string values

	 * @param aValueSet, set of unique values

	 * @param aDocument

	 * @return nodeList

	 */

	private NodeList toNodeList(Set<String> aValueSet, Document aDocument){

		DocumentFragment dFrag = aDocument.createDocumentFragment();

		Element root = aDocument.createElement("root");

		dFrag.appendChild(root);

		for (String value: aValueSet){

			Element eVal = aDocument.createElement("value");

			eVal.setTextContent(value);

			root.appendChild(eVal);

		}

		return dFrag.getChildNodes();

	}

	/**

	 * extract text from a given node using a list of given xpath expressions

	 * @param aXpathExprList

	 * @param aInput

	 * @param aXpath

	 * @return list of strings

	 * @throws XPathExpressionException

	 * @throws ParserConfigurationException 

	 */

	private List<String> extractText(List<String> aXpathExprList, Node aInput, XPath aXpath) throws XPathExpressionException, ParserConfigurationException{

		List<String> resultList = new LinkedList<String>();

		for (String xpathExpr: aXpathExprList){

			NodeList nodeList = (NodeList)aXpath.evaluate(xpathExpr, aInput, XPathConstants.NODESET);

			log.debug("extract text: nodelist length: " + nodeList.getLength()); 

			for (int i = 0; i < nodeList.getLength(); i++){

				resultList.add(nodeList.item(i).getTextContent());

			}

		}

		return resultList;

	}

}

/**

 * 

 */

package eu.dnetlib.data.collective.transformation.engine.functions;

/**

 * @author jochen

 *

 */

public class ProcessingException extends Exception {

	/**

	 * 

	 */

	private static final long serialVersionUID = -8648116731979859467L;

	/**

	 * 

	 */

	public ProcessingException() {

		super();

	}

	/**

	 * @param arg0

	 */

	public ProcessingException(String arg0) {

		super(arg0);

	}

	/**

	 * @param arg0

	 */

	public ProcessingException(Throwable arg0) {

		super(arg0);

	}

	/**

	 * @param arg0

	 * @param arg1

	 */

	public ProcessingException(String arg0, Throwable arg1) {

		super(arg0, arg1);

	}

}

/**

 * 

 */

package eu.dnetlib.data.collective.transformation.engine.functions;

import java.util.List;

import prototype.Person;

/**

 * @author jochen

 *

 */

public class PersonVocabulary extends Vocabulary{

	@Override

	public String encoding(List<String> keys)throws ProcessingException{

		Person p;

		String result = "";

		for (String input: keys){

			p = new Person(input);

			result = p.getNormalisedFullname();

		}

		return result;

	}

}

package eu.dnetlib.data.collective.transformation.engine;

import java.io.StringReader;

import java.util.HashMap;

import java.util.Iterator;

import java.util.LinkedHashMap;

import java.util.LinkedList;

import java.util.List;

import java.util.Map;

import javax.xml.xpath.XPath;

import javax.xml.xpath.XPathConstants;

import javax.xml.xpath.XPathExpressionException;

import javax.xml.xpath.XPathFactory;

import net.sf.saxon.instruct.TerminationException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.core.io.Resource;

import org.w3c.dom.Node;

import org.xml.sax.InputSource;

import eu.dnetlib.common.profile.ResourceDao;

import eu.dnetlib.data.collective.transformation.IDatabaseConnector;

import eu.dnetlib.data.collective.transformation.TransformationException;

import eu.dnetlib.data.collective.transformation.VocabularyRegistry;

import eu.dnetlib.data.collective.transformation.core.xsl.ext.TransformationFunctionProxy;

import eu.dnetlib.data.collective.transformation.engine.core.ITransformation;

import eu.dnetlib.data.collective.transformation.engine.functions.Convert;

// import eu.dnetlib.data.collective.transformation.engine.functions.Dblookup;

import eu.dnetlib.data.collective.transformation.engine.functions.Extract;

import eu.dnetlib.data.collective.transformation.engine.functions.IFeatureExtraction;

import eu.dnetlib.data.collective.transformation.engine.functions.ProcessingException;

import eu.dnetlib.data.collective.transformation.engine.functions.RegularExpression;

import eu.dnetlib.data.collective.transformation.engine.functions.RetrieveValue;

import eu.dnetlib.data.collective.transformation.engine.functions.RetrieveValue.FUNCTION;

import eu.dnetlib.data.collective.transformation.rulelanguage.Argument;

import eu.dnetlib.data.collective.transformation.rulelanguage.Argument.Type;

import eu.dnetlib.data.collective.transformation.rulelanguage.util.FunctionCall;

import eu.dnetlib.data.collective.transformation.utils.BlacklistConsumer;

/**

 * @author jochen

 *

 */

public class SimpleTransformationEngine {

	private static Log log = LogFactory.getLog(SimpleTransformationEngine.class);

	private ITransformation transformation;

	private VocabularyRegistry vocabularyRegistry;

	private IDatabaseConnector databaseConnector;

	private ResourceDao resourceDao;

	private IFeatureExtraction featureExtraction;

	private final List<String> mdRecords = new LinkedList<String>();

	private long totalTransformedRecords = 0;

	private long totalIgnoredRecords = 0;

	private String mappingFile;

	private boolean stylesheetParamsCalculated = false;

	private boolean preprocessingDone = false;

	private Map<String, String> stylesheetParams = new LinkedHashMap<String, String>();

	private Resource blacklistApi;

	private List<String> blacklistedRecords = new LinkedList<String>();

	/**

	 * execute any preprocessings declared in the transformation script prior starting the transformation of records

	 */

	public void preprocess(String dataSourceId) {

		for (Map<String, String> preprocMap : this.transformation.getRuleLanguageParser().getPreprocessings()) {

			Iterator<String> it = preprocMap.keySet().iterator();

			while (it.hasNext()) {

				String function = it.next();

//				if (function.equals("dblookup")) {

//					Dblookup fun = new Dblookup();

//					fun.setDbConnector(databaseConnector);

//					try {

//						log.debug("preprocessingMap value: " + preprocMap.get(function));

//						TransformationFunctionProxy.getInstance().setLookupRecord(fun.getResults(preprocMap.get(function)));

//					} catch (Exception e) {

//						log.debug(e.getMessage());

//						throw new IllegalStateException(e);

//					}

//				}

				if (function.equals("blacklist")) {

					BlacklistConsumer bc = new BlacklistConsumer();

					try{

						blacklistedRecords = bc.getBlackList(blacklistApi.getURL() + dataSourceId);						

					}catch(Exception e){

						throw new IllegalStateException("error in preprocess: " + e.getMessage());

					}

				}

			}

		}

		log.debug("preprocessing done.");

	}

	/**

	 * check if blacklistedRecords exist and if so check if the current record is blacklisted by its objIdentifier

	 * @param aRecord

	 * @return

	 * @throws XPathExpressionException

	 * @throws ProcessingException

	 */

	private boolean isBlacklistRecord(String aRecord){

		if (blacklistedRecords.size() == 0) return false;		

		XPath xpath = XPathFactory.newInstance().newXPath();

		try{

			Node root = (Node) xpath.evaluate("/", new InputSource(new StringReader(aRecord)), XPathConstants.NODE);

			String objId = xpath.evaluate("//*[local-name()='objIdentifier']", root);

			if (blacklistedRecords.contains(objId)) return true;			

		}catch(Exception e){

			throw new IllegalStateException("error in isBlacklistRecord: " + e.getMessage());

		}		

		return false;

	}

	/**

	 * transforms a source record

	 *

	 * @param sourceRecord

	 *            the record to transform

	 * @return transformed record

	 */

	public String transform(final String sourceRecord) {

		List<String> objectRecords = new LinkedList<String>();

		objectRecords.add(sourceRecord);

		int index = 0;

		mdRecords.clear();

		initTransformationFunction();

		if (!stylesheetParamsCalculated) {

			try{

				calculateStylesheetParams(sourceRecord);

			}catch(Exception e){

				throw new IllegalStateException("error in calculateStylesheetParams" + e.getMessage());

			}

		}

		if (!preprocessingDone){

			// xpath sourceRecord dataSourceid

			preprocess(stylesheetParams.get("varBlacklistDataSourceId"));

			preprocessingDone = true;

		}

		if (isBlacklistRecord(sourceRecord)){

			try{

				mdRecords.add(transformation.transformRecord(sourceRecord, ITransformation.XSLSyntaxcheckfailed));

			}catch(Exception e){

				log.fatal(sourceRecord);

				throw new IllegalStateException(e);				

			}

		}else if (!transformation.getRuleLanguageParser().isXslStylesheet()) {

			// iterate over all rules which are functionCalls

			log.debug("functionCalls size: " + transformation.getRuleLanguageParser().getFunctionCalls().size());

			for (FunctionCall functionCall : transformation.getRuleLanguageParser().getFunctionCalls()) {

				preprocess(objectRecords, functionCall);

			}

			for (String record : objectRecords) {

				// log.debug(record);

				try {

					log.debug("now run transformation for record with index: " + index);

					try{

						String transformedRecord = transformation.transformRecord(record, index);

						mdRecords.add(transformedRecord);

					} catch (TerminationException e){

						log.debug("record transformation terminated.");

						String failedRecord = transformation.transformRecord(record, ITransformation.XSLSyntaxcheckfailed);

						log.debug(failedRecord);

						totalIgnoredRecords++;

						mdRecords.add(failedRecord);

					}

				} catch (TransformationException e) {

					log.error(sourceRecord);

					throw new IllegalStateException(e);

				}

				index++;

			}

		} else {

			for (String record : objectRecords) {

				// test for init params and assign values

				try {

					log.debug("now run transformation for record with index: " + index);

					try{

						String transformedRecord = transformation.transformRecord(record, stylesheetParams);

						mdRecords.add(transformedRecord);

					}catch(TerminationException e){

						String failedRecord = transformation.transformRecord(record, ITransformation.XSLSyntaxcheckfailed);

						totalIgnoredRecords++;

						log.debug(failedRecord);

						mdRecords.add(failedRecord);

					}

				} catch (TransformationException e) {

					log.error(sourceRecord);

					throw new IllegalStateException(e);

				}

				index++;

			}

		}

		totalTransformedRecords = totalTransformedRecords + mdRecords.size();

		log.debug("objRecordSize: " + objectRecords.size() + ", mdRecordSize: " + mdRecords.size() + ", ignoredRecordSize: " + totalIgnoredRecords);

		return mdRecords.get(0);

	}

	private void calculateStylesheetParams(final String aRecord) throws XPathExpressionException, ProcessingException {

		stylesheetParamsCalculated = true;

		XPath xpath = XPathFactory.newInstance().newXPath();

		Node root = (Node) xpath.evaluate("/", new InputSource(new StringReader(aRecord)), XPathConstants.NODE);

		String datasourcePrefix = xpath.evaluate("//*[local-name()='datasourceprefix']", root);

		String profileXquery = "collection('/db/DRIVER/RepositoryServiceResources')//RESOURCE_PROFILE[.//EXTRA_FIELDS/FIELD[key=\"NamespacePrefix\"][value=\"" + datasourcePrefix + "\"]]";

		//String repositoryId = xpath.evaluate("//*[local-name()='repositoryId']", root);

		log.debug("profileXquery: " + profileXquery);

		// static $varDatasourceid = getValue(PROFILEFIELD, [xpath:"//dri:repositoryId",

		// xpath:"//EXTRA_FIELDS/FIELD[key='OpenAireDataSourceId']/value"]);

		RetrieveValue retrieveValue = new RetrieveValue();

		retrieveValue.setResourceDao(resourceDao);

		List<Argument> argList = new LinkedList<Argument>();

		argList.add(new Argument(Type.VALUE, profileXquery));

		Argument argXpath = new Argument(Type.INPUTFIELD, "//OFFICIAL_NAME");

		argList.add(argXpath);

		String varOfficialName = retrieveValue.executeSingleValue(FUNCTION.PROFILEFIELD.toString(), argList, null, new HashMap<String, String>());

		stylesheetParams.put("varOfficialName", varOfficialName);

		argList.remove(argXpath);

		argXpath = new Argument(Type.INPUTFIELD, "//EXTRA_FIELDS/FIELD[key='OpenAireDataSourceId']/value");

		argList.add(argXpath);

		String varDataSourceId = retrieveValue.executeSingleValue(FUNCTION.PROFILEFIELD.toString(), argList, null, new HashMap<String, String>());

		stylesheetParams.put("varDataSourceId", varDataSourceId);

		argList.remove(argXpath);

		argXpath = new Argument(Type.INPUTFIELD, "//CONFIGURATION/DATASOURCE_TYPE");

		argList.add(argXpath);

		String varDsType = retrieveValue.executeSingleValue(FUNCTION.PROFILEFIELD.toString(), argList, null, new HashMap<String, String>());

		stylesheetParams.put("varDsType", varDsType);

		argList.remove(argXpath);

		// if blacklist

		for (Map<String, String> preprocMap : this.transformation.getRuleLanguageParser().getPreprocessings()) {

			Iterator<String> it = preprocMap.keySet().iterator();

			while (it.hasNext()) {	

				String function = it.next();

				if (function.equals("blacklist")) {

					argXpath = new Argument(Type.INPUTFIELD, preprocMap.get(function)); // blacklistDataSourceIdXpath

					argList.add(argXpath);

					String varBlacklistDataSourceId = retrieveValue.executeSingleValue(FUNCTION.PROFILEFIELD.toString(), argList, null, new HashMap<String, String>());

					stylesheetParams.put("varBlacklistDataSourceId", varBlacklistDataSourceId);					

					argList.remove(argXpath);

				}

			}

		}

	}

	private void initTransformationFunction() {

		if (this.vocabularyRegistry == null) { throw new IllegalStateException("vocabularyReg is null"); }

		Convert convertFunction = new Convert();

		convertFunction.setVocabularyRegistry(this.vocabularyRegistry);

		TransformationFunctionProxy.getInstance().setConvertFunction(convertFunction);

	}

	/**

	 * preprocesses function if function is configured resp.

	 *

	 * @param records

	 *            list of object records

	 * @param aFunctionCall

	 */

	private void preprocess(final List<String> records, final FunctionCall aFunctionCall) {

		try {

			log.debug("preprocess");

			if (transformation.getRuleLanguageParser() == null) { throw new IllegalStateException("rulelanguageparser not initialised"); }

			if (transformation.getRuleLanguageParser().getNamespaceDeclarations() == null) { throw new IllegalStateException("nsDecl is null"); }

			PreProcessor preProc = new PreProcessor();

			preProc.setConvertFunction(TransformationFunctionProxy.getInstance().getConvertFunction());

			RetrieveValue retrieveValue = new RetrieveValue();

			retrieveValue.setResourceDao(resourceDao);

			preProc.setRetrieveFunction(retrieveValue);

			RegularExpression regExpr = new RegularExpression();

			preProc.setRegExprFunction(regExpr);

			TransformationFunctionProxy functionProxy = TransformationFunctionProxy.getInstance();

			preProc.setFunctionProxy(functionProxy);

			Extract extractFunction = new Extract();

			extractFunction.setFeatureExtraction(featureExtraction);

			preProc.setExtractFunction(extractFunction);

			if (aFunctionCall.doPreprocess() || aFunctionCall.isStatic()) {

				// log.debug("now call preprocess with: " + aFunctionCall.getExternalFunctionName() + " " + aFunctionCall.getUuid());

				preProc.preprocess(

						aFunctionCall,

						records,

						transformation.getRuleLanguageParser().getNamespaceDeclarations(),

						transformation.getStaticTransformationResults(),

						transformation.getJobProperties(),

						transformation.getRuleLanguageParser().getVariableMappingRules());

				// log.debug("preprocess end");

			} else {

				log.debug("skip preprocessing for function: " + aFunctionCall.getExternalFunctionName());

			}

		} catch (Exception e) {

			throw new IllegalStateException(e);

		}

	}

	/**

	 * @param transformation

	 *            the transformation to set

	 */

	public void setTransformation(final ITransformation transformation) {

		this.transformation = transformation;

	}

	/**

	 * @return the transformation

	 */

	public ITransformation getTransformation() {

		return transformation;

	}

	/**

	 * @param vocabularyRegistry

	 *            the vocabularyRegistry to set

	 */

	public void setVocabularyRegistry(final VocabularyRegistry vocabularyRegistry) {

		this.vocabularyRegistry = vocabularyRegistry;

	}

	/**

	 * @return the vocabularyRegistry

	 */

	public VocabularyRegistry getVocabularyRegistry() {

		return vocabularyRegistry;

	}

	/**

	 * @return the resourceDao

	 */

	public ResourceDao getResourceDao() {

		return resourceDao;

	}

	/**

	 * @param resourceDao

	 *            the resourceDao to set

	 */

	public void setResourceDao(final ResourceDao resourceDao) {

		this.resourceDao = resourceDao;

	}

	/**

	 * @param featureExtraction

	 *            the featureExtraction to set

	 */

	public void setFeatureExtraction(final IFeatureExtraction featureExtraction) {

		this.featureExtraction = featureExtraction;

	}

	/**

	 * @return the featureExtraction

	 */

	public IFeatureExtraction getFeatureExtraction() {

		return featureExtraction;

	}

	/**

	 * @return the databaseConnector

	 */

	public IDatabaseConnector getDatabaseConnector() {

		return databaseConnector;

	}

	/**

	 * @param databaseConnector

	 *            the databaseConnector to set

	 */

	public void setDatabaseConnector(final IDatabaseConnector databaseConnector) {

		this.databaseConnector = databaseConnector;

	}

	public long getTotalTransformedRecords() {

		return this.totalTransformedRecords;

	}

	public long getTotalIgnoredRecords() {

		return this.totalIgnoredRecords;

	}

	/**

	 * @return the mappingFile

	 */

	public String getMappingFile() {

		return mappingFile;

	}

	/**

	 * @param mappingFile

	 *            the mappingFile to set

	 */

	public void setMappingFile(final String mappingFile) {

		this.mappingFile = mappingFile;

	}

	public Resource getBlacklistApi() {

		return blacklistApi;

	}

	public void setBlacklistApi(Resource blacklistApi) {

		this.blacklistApi = blacklistApi;

	}

}

package eu.dnetlib.data.collective.transformation.core.schema.visitor;

import org.apache.commons.lang3.NotImplementedException;

import com.sun.xml.xsom.XSElementDecl;

import com.sun.xml.xsom.XSModelGroup;

import com.sun.xml.xsom.XSModelGroupDecl;

import com.sun.xml.xsom.XSParticle;

import com.sun.xml.xsom.XSWildcard;

import com.sun.xml.xsom.visitor.XSTermVisitor;

/**

 * @author jochen

 *

 */

public class XSTermVisitorImpl implements XSTermVisitor {

	private Visitor visitor;

	@Override

	public void elementDecl(XSElementDecl aElementDecl) {

		if (aElementDecl.isLocal()){

			  this.visitor.elementDecl(aElementDecl);

		}else{

			// ignore non local element declarations

		}

	}

	@Override

	public void modelGroup(XSModelGroup aModelGroup) {

		XSContentTypeVisitorImpl contentTypeVisitor = new XSContentTypeVisitorImpl();

		contentTypeVisitor.setVisitor(this.visitor);

		for (XSParticle p: aModelGroup.getChildren()){

			contentTypeVisitor.particle(p);

		}										

	}

	@Override

	public void modelGroupDecl(XSModelGroupDecl arg0) {

		throw new NotImplementedException("TODO modelGroupDecl");

	}

	@Override

	public void wildcard(XSWildcard arg0) {

		throw new NotImplementedException("TODO wildcard");

	}

	public void setVisitor(Visitor visitor) {

		this.visitor = visitor;

	}

	public Visitor getVisitor() {

		return visitor;

	}

}

package eu.dnetlib.data.collective.transformation.engine.core;

import java.io.Reader;

import java.io.StringReader;

import java.io.StringWriter;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.LinkedHashMap;

import java.util.List;

import java.util.Map;

import java.util.Properties;

import javax.xml.transform.OutputKeys;

import javax.xml.transform.Source;

import javax.xml.transform.Templates;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerConfigurationException;

import javax.xml.transform.TransformerException;

import javax.xml.transform.TransformerFactory;

import javax.xml.transform.stream.StreamResult;

import javax.xml.transform.stream.StreamSource;

import net.sf.saxon.FeatureKeys;

import net.sf.saxon.instruct.TerminationException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.DocumentHelper;

import org.dom4j.Element;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import org.springframework.core.io.Resource;

import eu.dnetlib.data.collective.transformation.TransformationException;

import eu.dnetlib.data.collective.transformation.core.schema.SchemaInspector;

import eu.dnetlib.data.collective.transformation.rulelanguage.RuleLanguageParser;

import eu.dnetlib.data.collective.transformation.rulelanguage.Rules;

import eu.dnetlib.data.collective.transformation.utils.NamespaceContextImpl;

/**

 * @author jochen

 *

 */

public class TransformationImpl implements

		ITransformation {

	private static final String rootElement = "record";

	private final Log log = LogFactory.getLog(TransformationImpl.class);

	private Document xslDoc;

	private SAXReader reader = new SAXReader();

	private Transformer transformer;

	private Transformer transformerFailed;

	protected RuleLanguageParser ruleLanguageParser;

	private StylesheetBuilder stylesheetBuilder;

	// cache static transformation results, valid for one transformation job

	private Map<String, String> staticResults = new LinkedHashMap<String, String>();

	private Map<String, String> jobConstantMap = new HashMap<String, String>();

	@javax.annotation.Resource(name="template")

	private Resource template;

	private Resource schema;

	private Source xsltSyntaxcheckFailed;

	/**

	 * initializes the transformation with the underlying XSL-template

	 */

	public void init(){

		try {

			xslDoc = reader.read(template.getInputStream());

			Resource xslResource = template.createRelative(XSLSyntaxcheckfailed);

			String systemId = xslResource.getURL().toExternalForm();

			xsltSyntaxcheckFailed = new StreamSource(xslResource.getInputStream(), systemId);

		} catch (Throwable e) {

			log.error("cannot initialize this transformation.", e);

			throw new IllegalStateException(e);

		}

	}

	public void addJobConstant(String aKey, String aValue){

		this.jobConstantMap.put(aKey, aValue);

	}

	/**

	 * creates a new Transformer object using a stylesheet based on the transformation rules

	 */

	public void configureTransformation()throws TransformerConfigurationException{

        final List<TransformerException> errorList = new ArrayList<TransformerException>();

        javax.xml.transform.ErrorListener listener = new javax.xml.transform.ErrorListener() {

			@Override

			public void warning(TransformerException exception) throws TransformerException {

				// TODO Auto-generated method stub

			}

			@Override

			public void fatalError(TransformerException exception) throws TransformerException {

				// TODO Auto-generated method stub

				errorList.add(exception);

				throw exception;

			}

			@Override

			public void error(TransformerException exception) throws TransformerException {

				// TODO Auto-generated method stub

			}

		};

		TransformerFactory factory = TransformerFactory.newInstance();

		factory.setAttribute(FeatureKeys.ALLOW_EXTERNAL_FUNCTIONS, Boolean.TRUE);

		factory.setErrorListener(listener);

		Templates templates = null;		

		try{

			if (this.ruleLanguageParser.isXslStylesheet()){

				templates = factory.newTemplates(new StreamSource(new StringReader(ruleLanguageParser.getXslStylesheet())));

			}else{

				templates = factory.newTemplates(new StreamSource(createStylesheet()));			

			}		

			transformer = templates.newTransformer();

			//((net.sf.saxon.Controller)transformer).setMessageEmitter(mw);

			transformer.setOutputProperty(OutputKeys.INDENT, "yes");

			transformer.setOutputProperty(OutputKeys.METHOD, "xml");

			transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");

			Templates templateFailed = factory.newTemplates(xsltSyntaxcheckFailed);

			transformerFailed = templateFailed.newTransformer();			

		}catch(TransformerConfigurationException e){

			if (!errorList.isEmpty()) {

                System.out.println(errorList.get(0).getMessageAndLocation()); // todo it seems the location information is not yet correct

                throw new TransformerConfigurationException(errorList.get(0).getMessageAndLocation());

			}else{

				throw e;

			}

		}		

		//((net.sf.saxon.Controller)transformerFailed).setMessageEmitter(mw);

	}

	/* (non-Javadoc)

	 * @see eu.dnetlib.data.collective.transformation.engine.core.ITransformation#transformRecord(java.lang.String, int)

	 */

	public String transformRecord(String record, int index)throws TerminationException, TransformationException{

		try {

			StreamSource s = new StreamSource(new StringReader(record));

			StringWriter writer = new StringWriter();

			StreamResult r = new StreamResult(writer);

			transformer.setParameter("index", index);

			transformer.transform(s , r);

			return writer.toString();

		}catch (TerminationException e) {

			log.debug(e.getLocalizedMessage());

			throw e;

		} catch (TransformerException e) {

			log.error(e);

			throw new TransformationException(e);

		}

	}

	public String transformRecord(String record, Map<String, String> parameters) throws TerminationException, TransformationException{

		try {

			StreamSource s = new StreamSource(new StringReader(record));

			StringWriter writer = new StringWriter();

			StreamResult r = new StreamResult(writer);

			for (String key: parameters.keySet()){

				transformer.setParameter(key, parameters.get(key));				

			}

			transformer.transform(s , r);

			return writer.toString();

		}catch (TerminationException e){

			log.debug(e.getLocalizedMessage());

			throw e;

		} catch (TransformerException e) {

			log.error(e);

			throw new TransformationException(e);

		}		

	}

	public String transformRecord(String record, String stylesheetName) throws TransformationException{

		if (!stylesheetName.equals(XSLSyntaxcheckfailed))

			throw new IllegalArgumentException("in TransformationImpl: stylesheetname " + stylesheetName + " is unsupported!" );

		try{

			StreamSource s = new StreamSource(new StringReader(record));

			StringWriter w = new StringWriter();

			StreamResult r = new StreamResult(w);

			transformerFailed.transform(s, r);

			return w.toString();			

		}catch (TransformerException e){

			log.error(e);

			throw new TransformationException(e);

		}

	}

	public String dumpStylesheet(){

		return xslDoc.asXML();

//		StringWriter writer = new StringWriter();

//		try {

//			Transformer tXsl = transformer; //.newTransformer();

//			tXsl.setOutputProperty(OutputKeys.INDENT, "yes");

//			tXsl.setOutputProperty(OutputKeys.METHOD, "xml");

//			tXsl.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");

//

//		StreamResult r = new StreamResult(writer);

//		Source s = new StreamSource(new StringReader(xslDoc.asXML()));

//			tXsl.transform(s, r);

//		} catch (TransformerException e) {

//			// TODO Auto-generated catch block

//			e.printStackTrace();

//		}

//		return writer.toString();

	}

	/**

	 * sets the XSL template

	 * @param template - resource to access the XSL template

	 */

	public void setTemplate(Resource template) {

		this.template = template;

	}

	/**

	 * @return the resource to access the XSL template

	 */

	public Resource getTemplate() {

		return template;

	}

	public void setRuleLanguageParser(RuleLanguageParser ruleLanguageParser) {

		this.ruleLanguageParser = ruleLanguageParser;

	}

	public RuleLanguageParser getRuleLanguageParser() {

		return ruleLanguageParser;

	}

	/**

	 * @param stylesheetBuilder the stylesheetBuilder to set

	 */

	public void setStylesheetBuilder(StylesheetBuilder stylesheetBuilder) {

		this.stylesheetBuilder = stylesheetBuilder;

	}

	/**

	 * @return the stylesheetBuilder

	 */

	public StylesheetBuilder getStylesheetBuilder() {

		return stylesheetBuilder;

	}

	/**

	 * @return the transformation rules as String object

	 */

	protected String getTransformationRules(){

		// add job-properties to the rules as variables

		for (String key: jobConstantMap.keySet()){

			Rules r = new Rules();

			r.setVariable(key);

			r.setConstant("'" + jobConstantMap.get(key) + "'");

			ruleLanguageParser.getVariableMappingRules().put(JOBCONST_DATASINKID, r);

		}

		if (this.stylesheetBuilder == null){

			// create DMF compliant stylesheet builder

			this.stylesheetBuilder = new StylesheetBuilder();

			this.stylesheetBuilder.setRuleLanguageParser(this.ruleLanguageParser);

			NamespaceContextImpl namespaceContext = new NamespaceContextImpl();

			for (String prefix: ruleLanguageParser.getNamespaceDeclarations().keySet()){

				namespaceContext.addNamespace(prefix, ruleLanguageParser.getNamespaceDeclarations().get(prefix));				

			}

			SchemaInspector inspector = new SchemaInspector();

			try {

				inspector.inspect(this.schema.getURL(), rootElement);

			} catch (Exception e) {

				throw new IllegalStateException(e);

			}

			this.stylesheetBuilder.setNamespaceContext(namespaceContext);

			this.stylesheetBuilder.setSchemaInspector(inspector);

		}

		return this.stylesheetBuilder.createTemplate();

	}

	/**

	 * creates a stylesheet from transformation rules;

	 * <p>don't call this method multiple times, unless transformation configuration changes, then re-init and configure transformation</p>

	 * @return the stylesheet

	 */

	private Reader createStylesheet(){

		try { 

			Document rulesDoc = DocumentHelper.parseText(getTransformationRules());

			for(String key: this.ruleLanguageParser.getNamespaceDeclarations().keySet()){

				xslDoc.getRootElement().addNamespace(key, this.ruleLanguageParser.getNamespaceDeclarations().get(key));				

			}

			@SuppressWarnings("unchecked")

			List<Node> nodes = rulesDoc.getRootElement().selectNodes("//xsl:template");

			@SuppressWarnings("unchecked")

			List<Node> varNodes = rulesDoc.getRootElement().selectNodes("/templateroot/xsl:param");

			for (Node node: varNodes){				

				xslDoc.getRootElement().add( ((Element)node).detach() );

			}

//			xslDoc.getRootElement().add(rulesDoc.getRootElement().selectSingleNode("//xsl:param[@name='var1']").detach());

			for (Node node: nodes){				

				xslDoc.getRootElement().add( ((Element)node).detach() ); //  (rulesDoc.getRootElement().aget);				

			}

		} catch (DocumentException e) {

			log.error("error in creating stylesheet: " + e);

			throw new IllegalStateException(e);

		}

		return new StringReader(xslDoc.asXML());

	}

	/**

	 * @param schema the schema to set

	 */

	public void setSchema(Resource schema) {

		this.schema = schema;

	}

	/**

	 * @return the schema

	 */

	public Resource getSchema() {

		return schema;

	}

	@Override

	public Map<String, String> getStaticTransformationResults() {

		return this.staticResults;

	}

	@Override

	public Map<String, String> getJobProperties() {

		// TODO Auto-generated method stub

		return this.jobConstantMap;

	}

	@Override

	public Properties getLogInformation() {

		// TODO Auto-generated method stub

		return null;

	}

}

package eu.dnetlib.data.collective.transformation.engine;

import java.io.StringReader;

import java.util.HashMap;

import java.util.LinkedList;

import java.util.List;

import java.util.Map;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.DocumentHelper;

import org.dom4j.Node;

import org.dom4j.XPath;

import org.dom4j.io.SAXReader;

import eu.dnetlib.data.collective.transformation.core.xsl.ext.TransformationFunctionProxy;

import eu.dnetlib.data.collective.transformation.engine.functions.Convert;

import eu.dnetlib.data.collective.transformation.engine.functions.Extract;

import eu.dnetlib.data.collective.transformation.engine.functions.ProcessingException;

import eu.dnetlib.data.collective.transformation.engine.functions.RegularExpression;

import eu.dnetlib.data.collective.transformation.engine.functions.RetrieveValue;

import eu.dnetlib.data.collective.transformation.rulelanguage.IRule;

import eu.dnetlib.data.collective.transformation.rulelanguage.Rules;

import eu.dnetlib.data.collective.transformation.rulelanguage.util.FunctionCall;

/**

 * @author jochen

 *

 */

public class PreProcessor {

	@SuppressWarnings("unused")

	private static final Log log = LogFactory.getLog(PreProcessor.class);

	private Convert convertFunction;

	private Extract extractFunction;

	private RetrieveValue retrieveFunction;

	private RegularExpression regExprFunction;

	private TransformationFunctionProxy functionProxy;

	private SAXReader reader = new SAXReader();

	private Map<String, String> nsMap = new HashMap<String, String>();

	/**

	 * pre-process output values from object records using a function call

	 * @param aFunctionCall function call object

	 * @param aObjectRecords list of object records

	 * @param aNamespaceMap map of namespace prefixes and uris

	 */

	public void preprocess( 

			FunctionCall aFunctionCall, 

			List<String> aObjectRecords, 

			Map<String, String> aNamespaceMap, 

			Map<String, String> aStaticResults, 

			Map<String, String> aJobProperties, 

			Map<String, IRule> aVarRules){

		this.nsMap = aNamespaceMap;

		FunctionResults functionResults = new FunctionResults();

		try {

			if (aFunctionCall.getExternalFunctionName().equals("extract")){

				String featureName = aFunctionCall.getParameters().get(Extract.paramNameFeature);

				functionResults.addAll(extractFunction.execute(aObjectRecords, featureName));

			}else{

				for (String objRecord: aObjectRecords){

					String result = null;

						if (aFunctionCall.getExternalFunctionName().equals("convert")){

							if (aFunctionCall.isStatic() && aStaticResults.containsKey(aFunctionCall.getUuid())){

								functionResults.add(aStaticResults.get(aFunctionCall.getUuid()));

							}else{

								String vocabName = aFunctionCall.getParameters().get(Convert.paramVocabularyName);

								String fieldExpr = aFunctionCall.getParameters().get(Convert.paramFieldValue);

								List<String> recordValues = getValuesFromRecord(objRecord, fieldExpr);

								if (aFunctionCall.isStatic())

									aStaticResults.put(aFunctionCall.getUuid(), convertFunction.executeSingleValue(vocabName, recordValues));

								else

									functionResults.add(convertFunction.executeAllValues(vocabName, recordValues));

							}

						}else if (aFunctionCall.getExternalFunctionName().equals("getValue")){

							if (aFunctionCall.isStatic() && aStaticResults.containsKey(aFunctionCall.getUuid()))

								functionResults.add(aStaticResults.get(aFunctionCall.getUuid()));

							else{

								String functionName = aFunctionCall.getParameters().get(RetrieveValue.paramFunctionName);

								result = retrieveFunction.executeSingleValue(functionName, aFunctionCall.getArguments(), objRecord, nsMap);

								functionResults.add(result);

								if (aFunctionCall.isStatic())

									aStaticResults.put(aFunctionCall.getUuid(), result);

							}

						}else if (aFunctionCall.getExternalFunctionName().equals("regExpr")){

							// TODO

							if (aFunctionCall.isStatic() && aStaticResults.containsKey(aFunctionCall.getUuid())){

								//log.debug("static functioncal; static result exist to compute regexpr: " + aFunctionCall.getParameters().get(RegularExpression.paramRegularExpr));

//								functionResults.add(aStaticResults.get(aFunctionCall.getUuid()));

							}else{

//								log.debug("static functioncal to compute regexpr: " + aFunctionCall.getParameters().get(RegularExpression.paramRegularExpr));

								String regularExpression = aFunctionCall.getParameters().get(RegularExpression.paramRegularExpr); //.replaceAll("'", "");

								String expression1 = aFunctionCall.getParameters().get(RegularExpression.paramExpr1);

								List<String> recordValues = null;

								// distinguish xpath-expr, jobConst, var

//								log.debug("expression1: " + expression1);

								if (aJobProperties.containsKey(expression1)){

									recordValues = new LinkedList<String>();

									recordValues.add(aJobProperties.get(expression1));

								}else{

									recordValues = getValuesFromRecord(objRecord, expression1);

								}

								String expression2 = aFunctionCall.getParameters().get(RegularExpression.paramExpr2);

								String replacement = "";

								if (aJobProperties.containsKey(expression2)){

									replacement = aJobProperties.get(expression2);

								}else if (aVarRules.containsKey(expression2)){

									Rules varRule = (Rules)aVarRules.get(expression2);

									replacement = varRule.getConstant().replace("'", ""); // currently limited to constant rules.

								}else {

									replacement = getValuesFromRecord(objRecord, expression2).get(0); // get the first available value

								}

								List<String> regExprResults = new LinkedList<String>();

								for (String fieldValue: recordValues){

									try {

										int lastSlash = regularExpression.lastIndexOf("/");

										String trailingOptions = regularExpression.substring(lastSlash);

										int replacementSlash = regularExpression.substring(0, lastSlash).lastIndexOf("/");

										String replacementFromExpression = regularExpression.substring(replacementSlash + 1, lastSlash);

										String newRegExpr = regularExpression.substring(0, replacementSlash + 1) + replacement + replacementFromExpression + trailingOptions; // ???

										result = regExprFunction.executeSingleValue(newRegExpr, fieldValue, replacement);

										regExprResults.add(result);

									} catch (ProcessingException e) {

										throw new IllegalStateException(e);

									}

//									regExprResults.add(regExprFunction.executeSingleValue(regularExpression, fieldValue, expression2));

								}

								functionResults.add(regExprResults);

								// assuming 1 result only

								if (aFunctionCall.isStatic()){

									aStaticResults.put(aFunctionCall.getUuid(), result);

								}

								// unsupported

//								if (aFunctionCall.isStatic()){

//									aStaticResults.put(aFunctionCall.getUuid(), result);

//								}

							}

						}

				}			

			}		

		} catch (ProcessingException e) {

			throw new IllegalStateException(e);

		} catch (DocumentException e) {

			throw new IllegalStateException(e);

		}

		functionProxy.setResults(aFunctionCall.getUuid(), functionResults);

	}

	public void setFunctionProxy(TransformationFunctionProxy functionProxy) {

		this.functionProxy = functionProxy;

	}

	public TransformationFunctionProxy getFunctionProxy() {

		return functionProxy;

	}

	public void setConvertFunction(Convert convertFunction) {

		this.convertFunction = convertFunction;

	}

	public Convert getConvertFunction() {

		return convertFunction;

	}

	/**

	 * @param retrieveFunction the retrieveFunction to set

	 */

	public void setRetrieveFunction(RetrieveValue retrieveFunction) {

		this.retrieveFunction = retrieveFunction;

	}

	/**

	 * @return the retrieveFunction

	 */

	public RetrieveValue getRetrieveFunction() {

		return retrieveFunction;

	}

	/**

	 * @return the regExprFunction

	 */

	public RegularExpression getRegExprFunction() {

		return regExprFunction;

	}

	/**

	 * @param regExprFunction the regExprFunction to set

	 */

	public void setRegExprFunction(RegularExpression regExprFunction) {

		this.regExprFunction = regExprFunction;

	}