D-Net

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-eagle-workflows/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "dnet-eagle-workflows"}

package eu.dnetlib.msro.eagle.ui;

import java.util.List;

import java.util.Map;

import javax.annotation.Resource;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import eu.dnetlib.data.download.rmi.DownloadPluginEnumerator;

import eu.dnetlib.msro.workflows.util.ValidNodeValuesFetcher;

public class DownloadPluginValues extends ValidNodeValuesFetcher {

	/** The download plugin enumerator. */

	@Resource

	private DownloadPluginEnumerator downloadPluginEnumerator;

	@Override

	protected List<DnetParamValue> obtainValues(final Map<String, String> params) throws Exception {

		return Lists.newArrayList(Iterables.transform(downloadPluginEnumerator.getAll().keySet(), new Function<String, DnetParamValue>() {

			@Override

			public DnetParamValue apply(final String input) {

				return new DnetParamValue(input, input);

			}

		}));

	}

	/**

	 * @return the downloadPluginEnumerator

	 */

	public DownloadPluginEnumerator getDownloadPluginEnumerator() {

		return downloadPluginEnumerator;

	}

	/**

	 * @param downloadPluginEnumerator

	 *            the downloadPluginEnumerator to set

	 */

	public void setDownloadPluginEnumerator(final DownloadPluginEnumerator downloadPluginEnumerator) {

		this.downloadPluginEnumerator = downloadPluginEnumerator;

	}

}

package eu.dnetlib.msro.eagle.workflows.nodes.composition;

import java.io.StringReader;

import java.util.Iterator;

import java.util.List;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.Element;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Required;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.resultset.MappedResultSetFactory;

import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;

import eu.dnetlib.enabling.resultset.client.utils.EPRUtils;

import eu.dnetlib.miscutils.functional.UnaryFunction;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

public class ComposeEprJobNode extends SimpleJobNode {

	private static final Log log = LogFactory.getLog(ComposeEprJobNode.class);

	private String mainEprParam;

	private String alternateEprParam;

	private String xpathToInjectionPoint;

	private String xpathToInjectedElements;

	private String outputEprParam;

	private MappedResultSetFactory mappedResultSetFactory;

	private ResultSetClientFactory resultSetClientFactory;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final W3CEndpointReference inputEpr = new EPRUtils().getEpr(token.getEnv().getAttribute(mainEprParam));

		final Iterator<String> alternateRsClient = resultSetClientFactory.getClient(token.getEnv().getAttribute(alternateEprParam)).iterator();

		final SAXReader reader = new SAXReader();

		final W3CEndpointReference outputEpr = mappedResultSetFactory.createMappedResultSet(inputEpr, new UnaryFunction<String, String>() {

			@Override

			public String evaluate(final String current) {

				try {

					Document main = reader.read(new StringReader(current));

					Document alternate = reader.read(new StringReader(alternateRsClient.next()));

					compose(main, alternate);

					return main.asXML();

				} catch (DocumentException e) {

					log.error("Error composing EPRs", e);

					throw new IllegalStateException(e);

				}

			}

			private void compose(final Document main, final Document alternate) {

				// Prepare injected fragments (source)

				List<Element> injectedElementList = alternate.selectNodes(getXpathToInjectedElements());

				// Retrieve injection point (destination)

				Element injectionPointElement = (Element) main.selectSingleNode(getXpathToInjectionPoint());

				if ((injectedElementList != null) && (injectionPointElement != null)) {

					Element parent = injectionPointElement.getParent();

					// injectionPointElement.detach();

					for (Element injectedElement : injectedElementList) {

						injectedElement.detach();

						// Move node around

						parent.add(injectedElement);

					}

				}

			}

		});

		token.getEnv().setAttribute(outputEprParam, outputEpr.toString());

		return Arc.DEFAULT_ARC;

	}

	public MappedResultSetFactory getMappedResultSetFactory() {

		return mappedResultSetFactory;

	}

	@Required

	public void setMappedResultSetFactory(

			final MappedResultSetFactory mappedResultSetFactory) {

		this.mappedResultSetFactory = mappedResultSetFactory;

	}

	public ResultSetClientFactory getResultSetClientFactory() {

		return resultSetClientFactory;

	}

	@Required

	public void setResultSetClientFactory(

			final ResultSetClientFactory resultSetClientFactory) {

		this.resultSetClientFactory = resultSetClientFactory;

	}

	public String getInputEprParam() {

		return mainEprParam;

	}

	public String getMainEprParam() {

		return mainEprParam;

	}

	public void setMainEprParam(final String mainEprParam) {

		this.mainEprParam = mainEprParam;

	}

	public String getAlternateEprParam() {

		return alternateEprParam;

	}

	public void setAlternateEprParam(final String alternateEprParam) {

		this.alternateEprParam = alternateEprParam;

	}

	public String getOutputEprParam() {

		return outputEprParam;

	}

	public void setOutputEprParam(final String outputEprParam) {

		this.outputEprParam = outputEprParam;

	}

	public String getXpathToInjectionPoint() {

		return xpathToInjectionPoint;

	}

	public void setXpathToInjectionPoint(final String xpathToInjectionPoint) {

		this.xpathToInjectionPoint = xpathToInjectionPoint;

	}

	public String getXpathToInjectedElements() {

		return xpathToInjectedElements;

	}

	public void setXpathToInjectedElements(final String xpathToInjectedElements) {

		this.xpathToInjectedElements = xpathToInjectedElements;

	}

}

package eu.dnetlib.msro.eagle.workflows.nodes.index;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

public class PrepareCreateIndexJobNode extends SimpleJobNode {

	private static final Log log = LogFactory.getLog(PrepareCreateIndexJobNode.class);

	private String layout;

	private String format;

	private String interpretation;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		log.info("Preparing env for CreateIndexJobNode");

		token.getEnv().setAttribute("layout", layout);

		token.getEnv().setAttribute("format", format);

		token.getEnv().setAttribute("interpretation", interpretation);

		return null;

	}

	public String getLayout() {

		return layout;

	}

	public void setLayout(final String layout) {

		this.layout = layout;

	}

	public String getFormat() {

		return format;

	}

	public void setFormat(final String format) {

		this.format = format;

	}

	public String getInterpretation() {

		return interpretation;

	}

	public void setInterpretation(final String interpretation) {

		this.interpretation = interpretation;

	}

}

package eu.dnetlib.msro.eagle.workflows.nodes.transform.tmid;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.resultset.MappedResultSetFactory;

import eu.dnetlib.enabling.resultset.client.utils.EPRUtils;

import eu.dnetlib.msro.rmi.MSROException;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

public class TrismegistosInjectionJobNode extends SimpleJobNode {

	private String inputEprParam;

	private String outputEprParam;

	private String tmIdListPath;

	private MappedResultSetFactory mappedResultSetFactory;

	@Override

	protected String execute(NodeToken token) throws Exception {

		final String inputEpr = token.getEnv().getAttribute(inputEprParam);

		if ((inputEpr == null) || inputEpr.isEmpty()) throw new MSROException("InputEprParam (" + inputEprParam + ") not found in ENV");

		final W3CEndpointReference epr = getMappedResultSetFactory().createMappedResultSet(new EPRUtils().getEpr(inputEpr), 

				new TrismegistosInjectionUnaryFunction(tmIdListPath));

		token.getEnv().setAttribute(outputEprParam, epr.toString());

		return Arc.DEFAULT_ARC;

	}

	public String getInputEprParam() {

		return inputEprParam;

	}

	public void setInputEprParam(String inputEprParam) {

		this.inputEprParam = inputEprParam;

	}

	public String getOutputEprParam() {

		return outputEprParam;

	}

	public void setOutputEprParam(String outputEprParam) {

		this.outputEprParam = outputEprParam;

	}

	public String getTmIdListPath() {

		return tmIdListPath;

	}

	public void setTmIdListPath(String tmIdListPath) {

		this.tmIdListPath = tmIdListPath;

	}

	public MappedResultSetFactory getMappedResultSetFactory() {

		return mappedResultSetFactory;

	}

	public void setMappedResultSetFactory(MappedResultSetFactory mappedResultSetFactory) {

		this.mappedResultSetFactory = mappedResultSetFactory;

	}

}

package eu.dnetlib.msro.eagle.workflows.nodes.transform.tmid;

import java.io.IOException;

import java.io.StringReader;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import javax.xml.stream.XMLInputFactory;

import javax.xml.stream.XMLStreamConstants;

import javax.xml.stream.XMLStreamException;

import javax.xml.stream.XMLStreamReader;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentHelper;

import org.dom4j.Element;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import org.springframework.core.io.FileSystemResource;

import eu.dnetlib.miscutils.functional.UnaryFunction;

import eu.dnetlib.miscutils.functional.hash.Hashing;

public class TrismegistosInjectionUnaryFunction implements UnaryFunction<String, String> {

	private static final Log log = LogFactory.getLog(TrismegistosInjectionUnaryFunction.class);

	private SAXReader reader;

	private static Map<String, List<String>> tmIdToLocalIdsMap;

	private static Map<String, String> localIdToTmIdMap;

	public TrismegistosInjectionUnaryFunction(final String tmIdListPath) throws XMLStreamException, IOException {

		this.reader = new SAXReader();

		tmIdToLocalIdsMap = new HashMap<String, List<String>>();

		localIdToTmIdMap = new HashMap<String, String>();

		FileSystemResource tmIds = new FileSystemResource(tmIdListPath);

		XMLInputFactory factory = XMLInputFactory.newInstance();

		XMLStreamReader reader = factory.createXMLStreamReader(tmIds.getInputStream());

		boolean isSafeToGetNextXmlElement = true;

		while (isSafeToGetNextXmlElement) {

			if (reader.getEventType() == XMLStreamConstants.START_ELEMENT) {

				if ("tmid".equals(reader.getLocalName())) {

					parseTmEvent(reader);

				}

			}

			if (reader.hasNext()) {

				reader.next();

			} else {

				isSafeToGetNextXmlElement = false;

				break;

			}

		}

	}

	private void parseTmEvent(final XMLStreamReader reader) throws XMLStreamException {

		String tmId = reader.getAttributeValue(null, "id");

		List<String> localIds = new ArrayList<String>();

		boolean isSafeToGetNextXmlElement = true;

		while (isSafeToGetNextXmlElement) {

			if (reader.getEventType() == XMLStreamConstants.END_ELEMENT) {

				break;

			} else {

				if (reader.getEventType() == XMLStreamConstants.START_ELEMENT) {

					if ("link".equals(reader.getLocalName())) {

						String localId = parseLinkEvent(reader);

						localIdToTmIdMap.put(localId, tmId);

						localIds.add(localId);

					}

				}

			}

			if (reader.hasNext()) {

				reader.next();

			} else {

				isSafeToGetNextXmlElement = false;

				break;

			}

		}

		tmIdToLocalIdsMap.put(tmId, localIds);

	}

	private String parseLinkEvent(final XMLStreamReader reader) throws XMLStreamException {

		String cp = reader.getAttributeValue(null, "cp");

		String val = "";

		boolean isSafeToGetNextXmlElement = true;

		while (isSafeToGetNextXmlElement) {

			if (reader.getEventType() == XMLStreamConstants.END_ELEMENT) {

				break;

			} else if (reader.getEventType() == XMLStreamConstants.CHARACTERS) {

				val = reader.getText();

			}

			if (reader.hasNext()) {

				reader.next();

			} else {

				isSafeToGetNextXmlElement = false;

				break;

			}

		}

		return cp + "::" + val;

	}

	@Override

	public String evaluate(final String inputXml) {

		try {

			Document doc = reader.read(new StringReader(inputXml));

			Node entityType = doc.selectSingleNode("//*[local-name()='entityType']");

			if (entityType == null) return inputXml;

			// TMid injection on Artifacts/Inscriptions

			if ("artifact".equals(entityType.getText())) {

				Node recordSourceInfo = doc.selectSingleNode("//*[local-name()='eagleObject']/*[local-name()='recordSourceInfo']");

				if (recordSourceInfo == null) return inputXml;

				String localId =

						recordSourceInfo.selectSingleNode("@providerAcronym").getText() + "::" + recordSourceInfo.selectSingleNode("./text()").getText();

				String tmId = localIdToTmIdMap.get(localId);

				if (tmId != null) {

					// TMid found! Prepare for injection..

					List<String> alternateIds = tmIdToLocalIdsMap.get(tmId);

					Element inscription = (Element) doc.selectSingleNode("//*[local-name()='inscription']");

					Element injectedHasTmId = prepareInjectedElement(tmId, alternateIds, localId);

					replaceElement(inscription, (Element) inscription.selectSingleNode("./*[local-name()='hasTmId']"), injectedHasTmId);

					return doc.asXML();

				} else // There is no TM ID for this localID. Just return the input XML here..

					return inputXml;

			}

			// TMid injection on other items with rel to Artifacts

			if ("visual".equals(entityType.getText()) || "documental".equals(entityType.getText())) {

				Node recordSourceInfo = doc.selectSingleNode("//*[local-name()='hasArtifact']/*[local-name()='recordSourceInfo']");

				if (recordSourceInfo == null) return inputXml;

				String localId =

						recordSourceInfo.selectSingleNode("@providerAcronym").getText() + "::" + recordSourceInfo.selectSingleNode("./text()").getText();

				String tmId = localIdToTmIdMap.get(localId);

				if (tmId != null) {

					// TMid found! Prepare for injection..

					List<String> alternateIds = tmIdToLocalIdsMap.get(tmId);

					Element inscription = (Element) doc.selectSingleNode("//*[local-name()='hasArtifact']");

					Element injectedHasTmId = prepareInjectedElement(tmId, alternateIds, localId);

					replaceElement(inscription, (Element) inscription.selectSingleNode("./*[local-name()='hasTmId']"), injectedHasTmId);

					return doc.asXML();

				} else // There is no TM ID for this localID. Just return the input string here..

					return inputXml;

			}

		} catch (Exception e) {

			log.error(inputXml);

			throw new IllegalStateException("Problems with Trismegistos injection", e);

		}

		return inputXml;

	}

	private Element prepareInjectedElement(final String tmId, final List<String> alternateIds, final String localId) {

		Document fragment = DocumentHelper.createDocument();

		Element hasTmId = fragment.addElement("hasTmId");

		hasTmId.addElement("tmId").addText(tmId);

		for (String alternateId : alternateIds) {

			if (!localId.equals(alternateId)) {

				String[] tokens = alternateId.split("::");

				hasTmId.addElement("alternateId").addAttribute("providerAcronym", tokens[0]).addAttribute("localId", tokens[1])

						.setText(tokens[0] + "::" + Hashing.md5(tokens[1]) + "::artifact");

			}

		}

		return (Element) fragment.selectSingleNode("./*[local-name()='hasTmId']");

	}

	private void replaceElement(final Element parent, final Element oldElement, final Element newElement) {

		List parentContent = parent.content();

		int index = parentContent.indexOf(oldElement);

		parentContent.set(index, newElement);

	}

}

package eu.dnetlib.msro.eagle.workflows.nodes.transform.editions;

import java.io.File;

import java.io.IOException;

import java.io.StringReader;

import java.util.Map;

import javax.xml.transform.stream.StreamSource;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.Element;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import eu.dnetlib.miscutils.functional.UnaryFunction;

import eu.dnetlib.miscutils.functional.xml.ApplyXslt;

public class EditionXsltUnaryFunction implements UnaryFunction<String, String> {

	private static final Log log = LogFactory.getLog(EditionXsltUnaryFunction.class);

	private ApplyXslt xslt;

	private SAXReader reader;

	public EditionXsltUnaryFunction(final String xsltPath, Map<String, String> xsltParams) throws IOException {

		this.xslt = new ApplyXslt(new StreamSource(new File(xsltPath)), xsltPath, xsltParams);

		this.reader = new SAXReader();

	}

	/**

	 * xPath the <metadata> node and applies xslt 2.0 transformation

	 */

	@Override

	public String evaluate(String input) {

		try {

			Document doc = reader.read(new StringReader(input));

			Node result = doc.selectSingleNode("//*[local-name()='metadata']/*");

			String edition = xslt.evaluate(result.asXML());

			result.detach();

			((Element) doc.selectSingleNode("//*[local-name()='metadata']")).add(reader.read(new StringReader(edition)).getRootElement());

			return doc.asXML();

		} catch (DocumentException e) {

			log.error("Problem creating dom4j document from \n" + input, e);

			throw new RuntimeException(e);

		}

	}

}

package eu.dnetlib.msro.eagle.workflows.nodes.transform.editions;

import java.util.HashMap;

import java.util.Map;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.springframework.beans.factory.annotation.Required;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.resultset.MappedResultSetFactory;

import eu.dnetlib.enabling.resultset.client.utils.EPRUtils;

import eu.dnetlib.msro.rmi.MSROException;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

/**

 * This class is intended to process records coming in MDStore compliant form (with header) and apply the xslt located at

 * http://sourceforge.net/projects/epidoc/files/Example%20Stylesheets/ to the <metadata> blocks of each record.

 *

 * @author Andrea Mannocci

 *

 */

public class ApplyEditionXsltJobNode extends SimpleJobNode {

	private String inputEprParam;

	private String outputEprParam;

	private String xsltPath;

	private MappedResultSetFactory mappedResultSetFactory;

	private final Map<String, String> xsltParams = new HashMap<String, String>();

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final String inputEpr = token.getEnv().getAttribute(inputEprParam);

		if ((inputEpr == null) || inputEpr.isEmpty()) throw new MSROException("InputEprParam (" + inputEprParam + ") not found in ENV");

		for (String name : token.getFullEnv().getAttributeNames()) {

			xsltParams.put(name, token.getFullEnv().getAttribute(name));

		}

		for (String name : token.getEnv().getAttributeNames()) {

			xsltParams.put(name, token.getEnv().getAttribute(name));

		}

		xsltParams.putAll(parseJsonParameters(token));

		final W3CEndpointReference epr = mappedResultSetFactory.createMappedResultSet(new EPRUtils().getEpr(inputEpr),

				new EditionXsltUnaryFunction(xsltPath, xsltParams));

		token.getEnv().setAttribute(outputEprParam, epr.toString());

		return Arc.DEFAULT_ARC;

	}

	public String getInputEprParam() {

		return inputEprParam;

	}

	public void setInputEprParam(final String inputEprParam) {

		this.inputEprParam = inputEprParam;

	}

	public String getOutputEprParam() {

		return outputEprParam;

	}

	public void setOutputEprParam(final String outputEprParam) {

		this.outputEprParam = outputEprParam;

	}

	public String getXsltPath() {

		return xsltPath;

	}

	public void setXsltPath(final String xsltPath) {

		this.xsltPath = xsltPath;

	}

	public MappedResultSetFactory getMappedResultSetFactory() {

		return mappedResultSetFactory;

	}

	@Required

	public void setMappedResultSetFactory(

			final MappedResultSetFactory mappedResultSetFactory) {

		this.mappedResultSetFactory = mappedResultSetFactory;

	}

}

package eu.dnetlib.msro.eagle.workflows.nodes.backlink.translations;

import java.io.StringReader;

import java.util.Iterator;

import java.util.List;

import java.util.Map;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.Element;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import eu.dnetlib.miscutils.functional.UnaryFunction;

public class TranslationBacklinkUnaryFunction implements UnaryFunction<String, String> {

	private static final Log log = LogFactory.getLog(TranslationBacklinkUnaryFunction.class);

	final private SAXReader reader;

	final private Map<String, List<TranslationInfo>> eagleIdToTranslationsInfo;

	public TranslationBacklinkUnaryFunction(final Iterator<String> translationIterator) {

		reader = new SAXReader();

		eagleIdToTranslationsInfo = Maps.newHashMap();

		while (translationIterator.hasNext()) {

			String translationRecord = translationIterator.next();

			try {

				processTranslationRecord(translationRecord, eagleIdToTranslationsInfo);

			} catch (DocumentException e) {

				log.error("Error during the pre-processing of MediaWiki translations", e);

				throw new RuntimeException(e);

			}

		}

	}

	private void processTranslationRecord(final String translationRecord, final Map<String, List<TranslationInfo>> eagleIdToTranslations)

			throws DocumentException {

		Document doc = reader.read(new StringReader(translationRecord));

		List<Node> eagleIdList = doc.selectNodes("//*[local-name()='hasArtifact']/*[local-name()='dnetResourceIdentifier']");

		List<TranslationInfo> translationInfoList;

		for (Node e : eagleIdList) {

			String[] tokens = e.getText().split("::");

			String eagleId = tokens[0] + "::" + tokens[1];

			// Fetch or create from scratch che List of EagleId

			if (eagleIdToTranslations.containsKey(eagleId)) {

				translationInfoList = eagleIdToTranslations.get(eagleId);

			} else {

				translationInfoList = Lists.newArrayList();

			}

			// Prepare TranslationInfo to append

			TranslationInfo info = new TranslationInfo();

			info.setDnetResourceIdentifier(doc.valueOf("//*[local-name()='eagleObject']/*[local-name()='dnetResourceIdentifier']"));

			info.setProviderAcronym(doc.valueOf("//*[local-name()='eagleObject']/*[local-name()='recordSourceInfo']/@providerAcronym"));

			info.setProviderName(doc.valueOf("//*[local-name()='eagleObject']/*[local-name()='recordSourceInfo']/@providerName"));

			info.setLandingPage(doc.valueOf("//*[local-name()='eagleObject']/*[local-name()='recordSourceInfo']/@landingPage"));

			info.setLocalId(doc.valueOf("//*[local-name()='eagleObject']/*[local-name()='recordSourceInfo']"));

			info.setText(doc.valueOf("//*[local-name()='translation']/*[local-name()='text']"));

			info.setLang(doc.valueOf("//*[local-name()='translation']/*[local-name()='text']/@lang"));

			translationInfoList.add(info);

			eagleIdToTranslations.put(eagleId, translationInfoList);

		}

	}

	@Override

	public String evaluate(final String record) {

		try {

			Document doc = reader.read(new StringReader(record));

			String eagleObjId = doc.valueOf("//*[local-name()='eagleObject']/*[local-name()='dnetResourceIdentifier']");

			String[] tokens = eagleObjId.split("::");

			eagleObjId = tokens[0] + "::" + tokens[1];

			if (!eagleIdToTranslationsInfo.containsKey(eagleObjId)) return record;

			Element injectionPoint = (Element) doc.selectSingleNode("//*[local-name()='inscription']");

			if (injectionPoint == null) {

				injectionPoint = (Element) doc.selectSingleNode("//*[local-name()='transcription']");

				if (injectionPoint == null) {

					injectionPoint = (Element) doc.selectSingleNode("//*[local-name()='visualRepresentation']");

				}

			}

			List<TranslationInfo> translationsInfoList = eagleIdToTranslationsInfo.get(eagleObjId);

			for (TranslationInfo translationInfo : translationsInfoList) {

				String fragment = HasTranslationFragment.generateFragment(translationInfo);

				Document fragmentDoc = new SAXReader().read(new StringReader(fragment));

				injectionPoint.add(fragmentDoc.getRootElement());

			}

			return doc.asXML();

		} catch (DocumentException e) {

			throw new RuntimeException(e);

		}

	}

}

package eu.dnetlib.msro.eagle.workflows.nodes.backlink.translations;

public class TranslationInfo {

	private String dnetResourceIdentifier;

	private String providerName;

	private String providerAcronym;

	private String landingPage;

	private String localId;

	private String text;

	private String lang;

	public String getDnetResourceIdentifier() {

		return dnetResourceIdentifier;

	}

	public void setDnetResourceIdentifier(final String dnetResourceIdentifier) {

		this.dnetResourceIdentifier = dnetResourceIdentifier;

	}

	public String getProviderName() {

		return providerName;

	}

	public void setProviderName(final String providerName) {

		this.providerName = providerName;

	}

	public String getProviderAcronym() {

		return providerAcronym;

	}

	public void setProviderAcronym(final String providerAcronym) {

		this.providerAcronym = providerAcronym;

	}

	public String getLandingPage() {

		return landingPage;

	}

	public void setLandingPage(final String landingPage) {

		this.landingPage = landingPage;

	}

	public String getLocalId() {

		return localId;

	}

	public void setLocalId(final String localId) {

		this.localId = localId;

	}

	public String getText() {

		return text;

	}

	public void setText(final String text) {

		this.text = text;

	}

	public String getLang() {

		return lang;

	}

	public void setLang(final String lang) {

		this.lang = lang;

	}

}

package eu.dnetlib.msro.eagle.workflows.nodes.backlink.translations;

import java.io.IOException;

import org.antlr.stringtemplate.StringTemplate;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang.StringEscapeUtils;

public class HasTranslationFragment {

	static public String generateFragment(final TranslationInfo translationInfo) {

		String fragmentTemplate;

		try {

			fragmentTemplate =

					IOUtils.toString(HasTranslationFragment.class

							.getResourceAsStream("/eu/dnetlib/msro/eagle/workflows/backlink/translations/hasTranslation.xml.st"));

			StringTemplate st = new StringTemplate(fragmentTemplate);

			st.setAttribute("dnetResourceIdentifier", StringEscapeUtils.escapeXml(translationInfo.getDnetResourceIdentifier()));

			st.setAttribute("providerName", StringEscapeUtils.escapeXml(translationInfo.getProviderName()));

			st.setAttribute("providerAcronym", StringEscapeUtils.escapeXml(translationInfo.getProviderAcronym()));

			st.setAttribute("landingPage", StringEscapeUtils.escapeXml(translationInfo.getLandingPage()));

			st.setAttribute("localId", StringEscapeUtils.escapeXml(translationInfo.getLocalId()));

			st.setAttribute("text", StringEscapeUtils.escapeXml(translationInfo.getText()));

			st.setAttribute("lang", StringEscapeUtils.escapeXml(translationInfo.getLang()));

			return st.toString();

		} catch (IOException e) {

			throw new RuntimeException("HasTranslation template not found!", e);

		}

	}

}

package eu.dnetlib.msro.eagle.workflows.nodes.backlink.translations;

import java.util.Iterator;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.data.mdstore.MDStoreService;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.enabling.resultset.MappedResultSetFactory;

import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;

import eu.dnetlib.enabling.resultset.client.utils.EPRUtils;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

public class TranslationBacklinkJobNode extends SimpleJobNode {

	private static final Log log = LogFactory.getLog(TranslationBacklinkJobNode.class);

	private String inputEprParam;

	private String outputEprParam;

	private MappedResultSetFactory mappedResultSetFactory;

	private ResultSetClientFactory resultSetClientFactory;

	@Autowired

	private UniqueServiceLocator serviceLocator;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final W3CEndpointReference inputEpr = new EPRUtils().getEpr(token.getEnv().getAttribute(inputEprParam));

		// retrieve Translation mdStore

		String xQueryTranslation = "for $x in collection('/db/DRIVER/WorkflowDSResources/WorkflowDSResourceType') "

				+ "where $x//WORKFLOW_NAME = 'index' and $x//PARAM[./@name = 'providerName']/text() = 'EAGLE MediaWiki' "

				+ "return $x//PARAM[./@name='mdId']/text()";

		String translationStoreId = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xQueryTranslation).get(0);

		final W3CEndpointReference translationsEpr = serviceLocator.getService(MDStoreService.class).deliverMDRecords(translationStoreId, "", "", "");

		final Iterator<String> translationIterator = resultSetClientFactory.getClient(translationsEpr).iterator();

		final W3CEndpointReference epr = mappedResultSetFactory.createMappedResultSet(inputEpr, new TranslationBacklinkUnaryFunction(translationIterator));

		token.getEnv().setAttribute(outputEprParam, epr.toString());

		return Arc.DEFAULT_ARC;

	}

	public String getInputEprParam() {

		return inputEprParam;

	}

	public void setInputEprParam(final String inputEprParam) {

		this.inputEprParam = inputEprParam;

	}

	public String getOutputEprParam() {

		return outputEprParam;

	}

	public void setOutputEprParam(final String outputEprParam) {

		this.outputEprParam = outputEprParam;

	}

	public MappedResultSetFactory getMappedResultSetFactory() {

		return mappedResultSetFactory;

	}

	@Required

	public void setMappedResultSetFactory(final MappedResultSetFactory mappedResultSetFactory) {

		this.mappedResultSetFactory = mappedResultSetFactory;

	}

	public ResultSetClientFactory getResultSetClientFactory() {

		return resultSetClientFactory;

	}

	@Required

	public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {

		this.resultSetClientFactory = resultSetClientFactory;

	}

}

package eu.dnetlib.data.collector.plugins.http;

import java.util.Iterator;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

public class UbiEratLupaHttpCollectorPlugin extends AbstractCollectorPlugin {

	@Override

	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

			throws CollectorServiceException {

		final String baseUrl = interfaceDescriptor.getBaseUrl();

		if (baseUrl == null || baseUrl.isEmpty()) { throw new CollectorServiceException("Param 'baseurl' is null or empty"); }

		return new Iterable<String>() {

			@Override

			public Iterator<String> iterator() {

				return new UbiEratLupaIterator(baseUrl);

			}

		};

	}

}

package eu.dnetlib.data.collector.plugins.http;

import java.io.IOException;

import java.net.MalformedURLException;

import java.net.URL;

import java.util.Iterator;

import java.util.Queue;

import java.util.concurrent.PriorityBlockingQueue;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

public class UbiEratLupaIterator implements Iterator<String> {

	private static final Log log = LogFactory.getLog(UbiEratLupaIterator.class);

	private String baseUrl;

	private int offset = 0;

	private static final int LIMIT = 500;

	private boolean onLastPage;

	private SAXReader reader = new SAXReader();

	private Queue<String> queue = new PriorityBlockingQueue<String>();

	public UbiEratLupaIterator() { }

	public UbiEratLupaIterator(String baseurl) {

		this.baseUrl = baseurl;

		this.onLastPage = false;

	}

	@Override

	public boolean hasNext() {

		if (queue.size() == 0) {

			if(onLastPage) {

				return false;

			} else {

				refillQueue();

				if (queue.size() == 0)

					return false;

				else

					return true;

			}

		} else {

			return true;

		}

	}

	private void refillQueue() {

		String url = baseUrl + "?limit=" + LIMIT + "&offset=" + offset;

		log.info("HTTP GET: " + url);

		URL request;

		try {

			request = new URL(url);

			Document doc = reader.read(request.openStream());

			for (Object o : doc.selectNodes("//monument")) {

				queue.add(((Node) o).asXML());

			}

			if (queue.size() < LIMIT) {

				onLastPage = true;

			}

			offset = offset + LIMIT;

		} catch (MalformedURLException e) {

			throw new RuntimeException("Malformed URL ->  " + url, e);

		} catch (DocumentException e) {

			throw new RuntimeException("An error occurred while parsing the page fetched from " + url, e);

		} catch (IOException e) {

			throw new RuntimeException("An error occurred getting page " + url, e);

		}

	}

	@Override

	public String next() {

		return queue.remove();

	}

	@Override

	public void remove() { }

}

package eu.dnetlib.data.collector.plugins.mediawiki;

import java.util.Iterator;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

public class MediawikiCollectorPlugin extends AbstractCollectorPlugin {

	@Override

	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

			throws CollectorServiceException {

		final String baseUrl = interfaceDescriptor.getBaseUrl();

		final String apnamespace = interfaceDescriptor.getParams().get("apnamespace");

		if (baseUrl == null || baseUrl.isEmpty()) { throw new CollectorServiceException("Param 'baseurl' is null or empty"); }

		if (apnamespace == null || baseUrl.isEmpty()) { throw new CollectorServiceException("Param 'apnamespace' is null or empty"); }

		return new Iterable<String>() {

			@Override

			public Iterator<String> iterator() {

				return new MediawikiIterator(baseUrl, apnamespace);

			}

		};

	}

}

package eu.dnetlib.data.collector.plugins.mediawiki;

import java.net.URL;

import java.util.Iterator;

import java.util.Queue;

import java.util.concurrent.PriorityBlockingQueue;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.Element;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

public class MediawikiIterator implements Iterator<String> {

	private static final Log log = LogFactory.getLog(MediawikiIterator.class);

	private SAXReader reader = new SAXReader();

	private Queue<String> queue = new PriorityBlockingQueue<String>();

	private String baseUrl;

	private String apnamespace;

	private String apcontinue;

	private boolean started;

	//	These are the call templates used by this class

	//	http://www.eagle-network.eu/wiki/api.php?action=query&list=allpages&apnamespace=120&aplimit=10&format=xml

	//	http://www.eagle-network.eu/wiki/api.php?action=wbgetentities&ids=Q100|Q1000|Q1001|Q1002|Q1003|Q1004|Q1005|Q1006|Q1007|Q1008

	public MediawikiIterator() { }

	public MediawikiIterator(final String baseUrl, final String apnamespace) {

		this.baseUrl = baseUrl;

		this.apnamespace = apnamespace;

		this.started = false;

	}

	private void verifyStarted() {

		if (!this.started) {

			this.apcontinue = firstPage();

			this.started = true;

		}

	}

	@Override

	public boolean hasNext() {

		synchronized (queue) {

			verifyStarted();

			return !queue.isEmpty();

		}

	}

	@Override

	public String next() {

		synchronized (queue) {

			verifyStarted();

			final String res = queue.poll();

			while (queue.isEmpty() && (apcontinue != null) && !apcontinue.isEmpty()) {

				apcontinue = otherPages(apcontinue);

			}

			return res;

		}

	}

	@Override

	public void remove() { }

	private String firstPage() {

		String url = baseUrl + "?action=query&list=allpages&apnamespace=" + apnamespace + "&aplimit=50&format=xml";

		log.info("Downloading first page using url: " + url);

		return downloadPage(url);

	}

	private String otherPages(String apcontinue) {

		return downloadPage(baseUrl + "?action=query&list=allpages&apnamespace=" + apnamespace + "&aplimit=50&format=xml&apcontinue=" + apcontinue);

	}

	private String downloadPage(String url) {

		try {

			log.info("HTTP GET: " + url);

			URL request = new URL(url);

			Document doc = reader.read(request.openStream());

			String nextApContinue = doc.valueOf("//*[local-name()='query-continue']//*[local-name()='allpages']/@apcontinue");

			String pagesUrl = "?action=wbgetentities&format=xml&ids=";

			for (Object o : doc.selectNodes("//*[local-name()='p']")) {

				String[] titleTokens = ((Element) o).valueOf("@title").split(":");

				pagesUrl += titleTokens[1] + "|";

			}

			pagesUrl = pagesUrl.substring(0, pagesUrl.length()-1); //remove last pipe char

			URL pages = new URL(baseUrl + pagesUrl);

			doc = reader.read(pages.openStream());

			// extract single mediawiki entities and enqueue each one separately

			for (Object o : doc.selectNodes("//*[local-name()='entity']")) {

				queue.add(((Node) o).asXML());

			}

			System.out.println("Done");

			return nextApContinue;

		} catch (Exception e) {

			throw new RuntimeException("Error processing data from: " + url, e);

		}

	}

}

package eu.dnetlib.data.download.plugins;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

public class EagleDownloadPlugin implements DownloadPlugin {

	@Override

	public DownloadItem retrieveUrl(final DownloadItem item) {

		String url = item.getUrl();

		if ((url == null) || (url.trim().length() == 0)) return null;

		@SuppressWarnings("unchecked")

		List<String> urls = new Gson().fromJson(url, ArrayList.class);

		if ((urls == null) || (urls.size() == 0)) {

			item.setOriginalUrl(null);

			item.setUrl(null);

			item.setFileName(item.getIdItemMetadata());

			return item;

		}

		item.setOriginalUrl(urls.get(0));

		item.setUrl(urls.get(0));

		item.setFileName(item.getIdItemMetadata());

		return item;

	}

	@Override

	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> items) {

		return Iterables.transform(items, new Function<DownloadItem, DownloadItem>() {

			@Override

			public DownloadItem apply(final DownloadItem input) {

				return retrieveUrl(input);

			}

		});

	}

	@Override

	public String getPluginName() {

		return "EagleDownloadPlugin";

	}

	@Override

	public void setBasePath(final String arg0) {

		// TODO Auto-generated method stub

	}

	/**

	 * {@inheritDoc}

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getRegularExpression()

	 */

	@Override

	public List<String> getRegularExpression() {

		// TODO Auto-generated method stub

		return null;

	}

	/**

	 * {@inheritDoc}

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#setRegularExpression(java.util.List)

	 */

	@Override

	public void setRegularExpression(final List<String> regularExpression) {

		// TODO Auto-generated method stub

	}

}

package eu.dnetlib.data.provision.epidoc;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.cache.annotation.Cacheable;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

public class MdIdResolver {

	private static final Log log = LogFactory.getLog(MdIdResolver.class);

	@Autowired

	private UniqueServiceLocator serviceLocator;

	@Cacheable(value = "mdIds", key = "#acronym")

	public String resolveMdId(final String acronym) throws ISLookUpException {

		log.info("Resolving mdID for " + acronym + ". Cache not used.");

		String xQuery =

				String.format(

						"let $apiId:=/RESOURCE_PROFILE[.//DATASOURCE_ORIGINAL_ID='%s']//INTERFACE[@compliance='eagleMetadata']/@id/string() "

								+ "return /RESOURCE_PROFILE[.//WORKFLOW_NAME='collect' and .//PARAM[@name='api']=$apiId]//NODE[@name='STORE']//PARAM[@name='mdId']/string()",

								acronym);

		String mdId = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xQuery).get(0);

		return mdId;

	}

}

package eu.dnetlib.data.provision.epidoc;

import java.io.IOException;

import java.io.StringReader;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.stereotype.Controller;

import org.springframework.web.bind.annotation.PathVariable;

import org.springframework.web.bind.annotation.RequestMapping;

import eu.dnetlib.data.mdstore.DocumentNotFoundException;

import eu.dnetlib.data.mdstore.MDStoreService;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

@Controller

public class EpidocRecordProvider {

	private static final Log log = LogFactory.getLog(EpidocRecordProvider.class);

	@Autowired

	private UniqueServiceLocator serviceLocator;

	@Autowired

	private MdIdResolver resolver;

	@RequestMapping("/epidoc/{objIdentfier}")

	public void provideEpidoc(final HttpServletRequest request, final HttpServletResponse response, @PathVariable final String objIdentfier)

			throws IOException, DocumentException, ISLookUpException, MDStoreServiceException {

		log.info("Retrieve original Epidoc record id = " + objIdentfier);

		String repoAcronym = objIdentfier.split("::")[0];

		String record;

		try {

			record = serviceLocator.getService(MDStoreService.class).deliverRecord(resolver.resolveMdId(repoAcronym), objIdentfier);

		} catch (DocumentNotFoundException e) {

			response.getWriter().write("The record identifier is invalid or not present");

			return;

		} catch (IndexOutOfBoundsException e) {

			response.getWriter().write("The record identifier is invalid");

			return;

		}

		Document doc = new SAXReader().read(new StringReader(record));

		Node teiNode = doc.selectSingleNode("//*[local-name()='TEI']");

		if (teiNode != null) {

			response.setHeader("Content-Type", "text/xml; charset=UTF-8");

			response.getWriter().write(doc.selectSingleNode("//*[local-name()='TEI']").asXML());

		} else {

			response.getWriter().write("EpiDoc representation not available for this record");

		}

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<RESOURCE_PROFILE xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">

	<HEADER>

		<RESOURCE_IDENTIFIER

			value="dd5882b3-1a92-4e59-929c-f36b73854e57_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl" />

		<RESOURCE_TYPE value="WorkflowDSResourceType" />

		<RESOURCE_KIND value="WorkflowDSResources" />

		<RESOURCE_URI value="value3" />

		<DATE_OF_CREATION value="2006-05-04T18:13:51.0Z" />

	</HEADER>

	<BODY>

		<WORKFLOW_NAME>EpiDoc collection and transformation</WORKFLOW_NAME>

		<WORKFLOW_TYPE>REPO_HI</WORKFLOW_TYPE>

		<WORKFLOW_PRIORITY>20</WORKFLOW_PRIORITY>

		<CONFIGURATION start="manual">

			<NODE name="VerifyDatasource" type="VerifyDatasource" isStart="true">

				<DESCRIPTION>Verify if DS is pending</DESCRIPTION>

				<PARAMETERS>

					<PARAM name="expectedInterfaceTypologyPrefixes" managedBy="system" required="false" type="string">epidoc</PARAM>

					<PARAM name="expectedCompliancePrefixes" managedBy="system" required="false" type="string">eagleMetadata</PARAM>

				</PARAMETERS>

				<ARCS>

					<ARC to="createMetaWf"/>

					<ARC to="validateDs" name="validateDs"/>

				</ARCS>

			</NODE>

			<NODE name="validateDs" type="ValidateDatasource">

				<DESCRIPTION>Validate DS</DESCRIPTION>

				<PARAMETERS/>

				<ARCS>

					<ARC to="createMetaWf"/>

				</ARCS>

			</NODE>

			<NODE name="createMetaWf" type="RegisterMetaWf">

				<DESCRIPTION>Create MetaWorkflow</DESCRIPTION>

				<PARAMETERS>

					<PARAM name="wfName" managedBy="system" required="true" type="string">EpiDoc collection and transformation</PARAM>

				</PARAMETERS>

				<ARCS>

					<ARC to="createNative"/>

					<ARC to="createTransformed"/>

					<ARC to="createCleaned"/>

					<ARC to="prepareCreateIndex"/>

				</ARCS>

			</NODE>

			<NODE name="createNative" type="CreateMDStore">

				<DESCRIPTION>Create native store</DESCRIPTION>

				<PARAMETERS>

					<PARAM name="format" managedBy="system" required="true" type="string">GMF</PARAM>

					<PARAM name="interpretation" managedBy="system" required="true" type="string">native</PARAM>

					<PARAM name="layout" managedBy="system" required="true" type="string">store</PARAM>

					<PARAM name="outputPrefix" managedBy="system" required="true" type="string">coll_</PARAM>

				</PARAMETERS>

				<ARCS>

					<ARC to="updateMetaWf" />