D-Net

package eu.dnetlib.msro.workflows.nodes.info;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

import eu.dnetlib.msro.workflows.util.WorkflowsConstants;

public class SetProviderInfoJobNode extends SimpleJobNode {

	private String providerId;

	private String providerName;

	private String api;

	@Override

	protected String execute(NodeToken token) throws Exception {

		token.getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_ID, getProviderId());

		token.getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_NAME, getProviderName());

		token.getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_INTERFACE, getApi());

		token.getFullEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_ID, getProviderId());

		token.getFullEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_NAME, getProviderName());

		token.getFullEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_INTERFACE, getApi());

		token.getProcess().getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_ID, getProviderId());

		token.getProcess().getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_NAME, getProviderName());

		token.getProcess().getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_INTERFACE, getApi());

		return Arc.DEFAULT_ARC;

	}

	public String getProviderId() {

		return providerId;

	}

	public void setProviderId(String providerId) {

		this.providerId = providerId;

	}

	public String getProviderName() {

		return providerName;

	}

	public void setProviderName(String providerName) {

		this.providerName = providerName;

	}

	public String getApi() {

		return api;

	}

	public void setApi(String api) {

		this.api = api;

	}

}

package eu.dnetlib.msro.workflows.nodes.objectStore;

import java.util.List;

import java.util.Set;

import javax.annotation.Resource;

import org.springframework.beans.factory.annotation.Required;

import com.google.common.collect.Lists;

import com.google.common.collect.Sets;

import com.google.gson.Gson;

import com.google.gson.GsonBuilder;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

/**

 * The Class RetrieveMdStoreId is a job node used to retrieve the correct MDStore from which extract the url of the file to download.

 * metadata format and interpretation are injected as properties

 */

public class RetrieveMdStoreId extends SimpleJobNode {

	/** The metadata format. */

	private String metadataFormat;

	/** The interpretation. */

	private String interpretation;

	/** The provider id. */

	private String providerId;

	/** The service locator. */

	@Resource

	private UniqueServiceLocator serviceLocator;

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.msro.workflows.nodes.SimpleJobNode#execute(com.googlecode.sarasvati.NodeToken)

	 */

	@Override

	protected String execute(final NodeToken token) throws Exception {

		String workflowQuery =

				"for $x in collection('/db/DRIVER/MetaWorkflowDSResources/MetaWorkflowDSResourceType') where($x//DATAPROVIDER/@id='%s') return  distinct-values($x//WORKFLOW/@id/string())";

		List<String> result = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(String.format(workflowQuery, providerId));

		if (result.size() == 0) { throw new RuntimeException("there is no mdStore Associated to the provider " + token.getEnv().getAttribute(getProviderId())); }

		Set<String> workflowIds = Sets.newHashSet(result);

		Set<String> metadataIds = getMdStores(workflowIds);

		Gson g = new GsonBuilder().disableHtmlEscaping().create();

		token.getEnv().setAttribute("mdId", g.toJson(metadataIds));

		token.getEnv().setAttribute("mdFormat", getMetadataFormat());

		return Arc.DEFAULT_ARC;

	}

	private Set<String> getMdStores(final Set<String> workflowsId) {

		try {

			String query = "//RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value='%s']//PARAM[./@category/string()='MDSTORE_ID']/text()";

			Set<String> mdStores = Sets.newHashSet();

			if (workflowsId == null) { return null; }

			for (String workflowId : workflowsId) {

				List<String> result = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(String.format(query, workflowId));

				Set<String> metadataIds = Sets.newHashSet(result);

				mdStores.addAll(getRightMetadataId(Lists.newArrayList(metadataIds)));

			}

			return mdStores;

		} catch (ISLookUpException e) {

			return null;

		}

	}

	/**

	 * Gets the right metadata id whith the format metadataFormat and interpretation interpretation

	 *

	 * @return the right metadata id

	 * @throws ISLookUpException

	 */

	private Set<String> getRightMetadataId(final Iterable<String> ids) throws ISLookUpException {

		String query =

				"let $x:=//RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value='%s'] return concat($x//METADATA_FORMAT/text(), '::<<>>::', $x//METADATA_FORMAT_INTERPRETATION/text())";

		Set<String> result = Sets.newHashSet();

		for (String id : ids) {

			List<String> results = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(String.format(query, id));

			if (results.size() > 0) {

				String[] values = results.get(0).split("::<<>>::");

				if (metadataFormat.equals(values[0]) && interpretation.equals(values[1])) {

					result.add(id);

				}

			}

		}

		return result;

	}

	/**

	 * Gets the interpretation.

	 *

	 * @return the interpretation

	 */

	public String getInterpretation() {

		return interpretation;

	}

	/**

	 * Sets the interpretation.

	 *

	 * @param interpretation

	 *            the interpretation to set

	 */

	@Required

	public void setInterpretation(final String interpretation) {

		this.interpretation = interpretation;

	}

	/**

	 * Gets the metadata format.

	 *

	 * @return the metadataFormat

	 */

	public String getMetadataFormat() {

		return metadataFormat;

	}

	/**

	 * Sets the metadata format.

	 *

	 * @param metadataFormat

	 *            the metadataFormat to set

	 */

	@Required

	public void setMetadataFormat(final String metadataFormat) {

		this.metadataFormat = metadataFormat;

	}

	/**

	 * Gets the provider id.

	 *

	 * @return the providerId

	 */

	public String getProviderId() {

		return providerId;

	}

	/**

	 * Sets the provider id.

	 *

	 * @param providerId

	 *            the providerId to set

	 */

	public void setProviderId(final String providerId) {

		this.providerId = providerId;

	}

}

package eu.dnetlib.msro.workflows.nodes.objectStore;

import javax.annotation.Resource;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

import com.google.common.collect.Iterables;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.resultset.IterableResultSetFactory;

import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;

import eu.dnetlib.enabling.resultset.client.utils.EPRUtils;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

import eu.dnetlib.msro.workflows.nodes.download.UrlExtractor;

// TODO: Auto-generated Javadoc

/**

 * The Class RetrieveURLSJobNode.

 */

public class RetrieveURLSJobNode extends SimpleJobNode {

	/** The epr param. */

	private String inputEprParam;

	/** The output epr param. */

	private String outputEprParam;

	/** The xpath. */

	private String xpath;

	/** The xpath metadata id. */

	private String xpathMetadataId;

	/** The xpath open access. */

	private String xpathOpenAccess;

	/** The xpath embargo date. */

	private String xpathEmbargoDate;

	/** The result set client factory. */

	@Autowired

	private ResultSetClientFactory resultSetClientFactory;

	/** The result set factory. */

	@Resource(name = "iterableResultSetFactory")

	private IterableResultSetFactory resultSetFactory;

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.msro.workflows.nodes.SimpleJobNode#execute(com.googlecode.sarasvati.NodeToken)

	 */

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final W3CEndpointReference inputEpr = (new EPRUtils()).getEpr(token.getEnv().getAttribute(inputEprParam));

		Iterable<String> input = resultSetClientFactory.getClient(inputEpr);

		Iterable<String> extractedUrls = Iterables.transform(input, new UrlExtractor(xpath, xpathMetadataId, xpathOpenAccess, xpathEmbargoDate));

		W3CEndpointReference eprUrls = resultSetFactory.createIterableResultSet(extractedUrls);

		token.getEnv().setAttribute(getOutputEprParam(), eprUrls.toString());

		return Arc.DEFAULT_ARC;

	}

	/**

	 * Gets the xpath.

	 *

	 * @return the xpath

	 */

	public String getXpath() {

		return xpath;

	}

	/**

	 * Sets the xpath.

	 *

	 * @param xpath

	 *            the xpath to set

	 */

	public void setXpath(final String xpath) {

		this.xpath = xpath;

	}

	/**

	 * Gets the xpath metadata id.

	 *

	 * @return the xpathMetadataId

	 */

	public String getXpathMetadataId() {

		return xpathMetadataId;

	}

	/**

	 * Sets the xpath metadata id.

	 *

	 * @param xpathMetadataId

	 *            the xpathMetadataId to set

	 */

	@Required

	public void setXpathMetadataId(final String xpathMetadataId) {

		this.xpathMetadataId = xpathMetadataId;

	}

	/**

	 * Gets the result set client factory.

	 *

	 * @return the resultSetClientFactory

	 */

	public ResultSetClientFactory getResultSetClientFactory() {

		return resultSetClientFactory;

	}

	/**

	 * Sets the result set client factory.

	 *

	 * @param resultSetClientFactory

	 *            the resultSetClientFactory to set

	 */

	public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {

		this.resultSetClientFactory = resultSetClientFactory;

	}

	/**

	 * Gets the result set factory.

	 *

	 * @return the resultSetFactory

	 */

	public IterableResultSetFactory getResultSetFactory() {

		return resultSetFactory;

	}

	/**

	 * Sets the result set factory.

	 *

	 * @param resultSetFactory

	 *            the resultSetFactory to set

	 */

	public void setResultSetFactory(final IterableResultSetFactory resultSetFactory) {

		this.resultSetFactory = resultSetFactory;

	}

	/**

	 * Gets the output epr param.

	 *

	 * @return the outputEprParam

	 */

	public String getOutputEprParam() {

		return outputEprParam;

	}

	/**

	 * Sets the output epr param.

	 *

	 * @param outputEprParam

	 *            the outputEprParam to set

	 */

	public void setOutputEprParam(final String outputEprParam) {

		this.outputEprParam = outputEprParam;

	}

	/**

	 * Gets the input epr param.

	 *

	 * @return the inputEprParam

	 */

	public String getInputEprParam() {

		return inputEprParam;

	}

	/**

	 * Sets the input epr param.

	 *

	 * @param inputEprParam

	 *            the inputEprParam to set

	 */

	public void setInputEprParam(final String inputEprParam) {

		this.inputEprParam = inputEprParam;

	}

	/**

	 * Gets the xpath open access.

	 *

	 * @return the xpath open access

	 */

	public String getXpathOpenAccess() {

		return xpathOpenAccess;

	}

	/**

	 * Sets the xpath open access.

	 *

	 * @param xpathOpenAccess

	 *            the new xpath open access

	 */

	public void setXpathOpenAccess(final String xpathOpenAccess) {

		this.xpathOpenAccess = xpathOpenAccess;

	}

	/**

	 * Gets the xpath embargo date.

	 *

	 * @return the xpath embargo date

	 */

	public String getXpathEmbargoDate() {

		return xpathEmbargoDate;

	}

	/**

	 * Sets the xpath embargo date.

	 *

	 * @param xpathEmbargoDate

	 *            the new xpath embargo date

	 */

	public void setXpathEmbargoDate(final String xpathEmbargoDate) {

		this.xpathEmbargoDate = xpathEmbargoDate;

	}

}

package eu.dnetlib.msro.workflows.nodes;

import javax.annotation.Resource;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.googlecode.sarasvati.Engine;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.enabling.tools.blackboard.BlackboardClientHandler;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJobImpl;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJobRegistry;

import eu.dnetlib.msro.workflows.nodes.blackboard.BlackboardWorkflowJobListener;

import eu.dnetlib.msro.workflows.util.WorkflowsConstants;

public abstract class BlackboardJobNode extends SarasvatiJobNode {

	@Resource

	private UniqueServiceLocator serviceLocator;

	/**

	 * logger.

	 */

	private static final Log log = LogFactory.getLog(BlackboardJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	/**

	 * blackboard handler.

	 */

	@Resource

	private BlackboardClientHandler blackboardClientHandler;

	/**

	 * blackboard job registry.

	 */

	@Resource

	private BlackboardJobRegistry jobRegistry;

	@Override

	public void execute(final Engine engine, final NodeToken token) {

		super.execute(engine, token);

		log.info("executing blackboard node");

		try {

			token.getEnv().setAttribute(WorkflowsConstants.BLACKBOARD_IS_BLACKBOARD, true);

			final String serviceId = obtainServiceId(token);

			if (StringUtils.isBlank(serviceId)) {

				token.getEnv().setAttribute(WorkflowsConstants.SYSTEM_HAS_FAILED, true);

				final String msg = "cannot locate target service profile: " + serviceId;

				token.getEnv().setAttribute(WorkflowsConstants.SYSTEM_ERROR, msg);

				log.error(msg);

				engine.complete(token, "failed");

				return;

			}

			final BlackboardJob job = blackboardClientHandler.newJob(serviceId);

			token.getEnv().setTransientAttribute(WorkflowsConstants.BLACKBOARD_JOB, job);

			token.getEnv().setAttribute(WorkflowsConstants.BLACKBOARD_SERVICE_ID, ((BlackboardJobImpl) job).getServiceId());

			prepareJob(job, token);

			jobRegistry.registerJobListener(job, generateBlackboardListener(engine, token));

			blackboardClientHandler.assign(job);

		} catch (final Throwable e) {

			token.getEnv().setAttribute(WorkflowsConstants.SYSTEM_HAS_FAILED, true);

			token.getEnv().setAttribute(WorkflowsConstants.SYSTEM_ERROR, "cannot prepare blackboard job: " + e);

			engine.complete(token, "failed");

			log.error("cannot prepare blackboard job", e);

		}

	}

	abstract protected String obtainServiceId(NodeToken token);

	abstract protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception;

	protected BlackboardWorkflowJobListener generateBlackboardListener(final Engine engine, final NodeToken token) {

		return new BlackboardWorkflowJobListener(engine, token);

	}

	public UniqueServiceLocator getServiceLocator() {

		return serviceLocator;

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-msro-service</artifactId>

	<packaging>jar</packaging>

	<version>3.0.9</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-msro-service/tags/dnet-msro-service-3.0.9</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>opensymphony</groupId>

			<artifactId>quartz</artifactId>

			<version>1.6.6</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-msro-service-api</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-resultset-service</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-data-utility-cleaner-rmi</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-resultset-client</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-download-service-rmi</artifactId>

			<version>[1.1.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-datasource-manager-rmi</artifactId>

			<version>[4.0.0,5.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-blackboard-common</artifactId>

			<version>[2.1.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-mongo-logging</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>com.googlecode</groupId>

			<artifactId>sarasvati</artifactId>

			<version>1.0.3</version>

		</dependency>

		<dependency>

			<groupId>com.googlecode</groupId>

			<artifactId>sarasvati-visual</artifactId>

			<version>1.0.3</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-modular-collector-service-rmi</artifactId>

			<version>[1.3.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-enabling-database-api</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-objectstore-rmi</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-data-transformation-service-rmi</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-data-provision-rmi</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-runtime</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>javax.mail</groupId>

			<artifactId>mail</artifactId>

			<version>1.4</version>

		</dependency>

		<dependency>

			<groupId>org.codehaus.groovy</groupId>

			<artifactId>groovy-all</artifactId>

			<version>2.1.6</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>joda-time</groupId>

			<artifactId>joda-time</artifactId>

			<version>2.3</version>

		</dependency>

	</dependencies>

	<properties>

		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

	</properties>

</project>

package eu.dnetlib.msro.workflows.nodes.db;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.database.rmi.DatabaseService;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;

import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;

import eu.dnetlib.msro.workflows.resultset.ProcessCountingResultSetFactory;

import eu.dnetlib.msro.workflows.util.ProgressProvider;

import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider;

public class UpdateDbJobNode extends BlackboardJobNode implements ProgressJobNode {

	private String db;

	private String dbParam;

	private String eprParam;

	private ProcessCountingResultSetFactory processCountingResultSetFactory;

	private ResultsetProgressProvider progressProvider;

	private static final Log log = LogFactory.getLog(UpdateDbJobNode.class);

	@Override

	protected String obtainServiceId(final NodeToken token) {

		return getServiceLocator().getServiceId(DatabaseService.class);

	}

	@Override

	protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {

		if (db == null || db.isEmpty()) {

			db = token.getEnv().getAttribute(getDbParam());

		}

		log.info("preparing blackboard job to update DB: " + db);

		final String epr = token.getEnv().getAttribute(getEprParam());

		this.progressProvider = processCountingResultSetFactory.createProgressProvider(token.getProcess(), epr);

		job.setAction("IMPORT");

		job.getParameters().put("db", db);

		job.getParameters().put("epr", this.progressProvider.getEpr().toString());

	}

	@Override

	public ProgressProvider getProgressProvider() {

		return progressProvider;

	}

	public String getDb() {

		return db;

	}

	public void setDb(final String db) {

		this.db = db;

	}

	public String getEprParam() {

		return eprParam;

	}

	public void setEprParam(final String eprParam) {

		this.eprParam = eprParam;

	}

	public ProcessCountingResultSetFactory getProcessCountingResultSetFactory() {

		return processCountingResultSetFactory;

	}

	@Required

	public void setProcessCountingResultSetFactory(final ProcessCountingResultSetFactory processCountingResultSetFactory) {

		this.processCountingResultSetFactory = processCountingResultSetFactory;

	}

	public String getDbParam() {

		return dbParam;

	}

	public void setDbParam(final String dbParam) {

		this.dbParam = dbParam;

	}

}

package eu.dnetlib.msro.workflows.metawf;

import java.io.IOException;

import java.util.Map;

import org.antlr.stringtemplate.StringTemplate;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang.StringEscapeUtils;

import org.springframework.core.io.ClassPathResource;

import org.springframework.core.io.Resource;

import com.google.common.base.Function;

import com.google.common.collect.Maps;

import eu.dnetlib.msro.workflows.util.WorkflowsConstants;

public class WorkflowProfileCreator {

	private static final Resource wfTemplate = new ClassPathResource("/eu/dnetlib/msro/workflows/templates/workflow.xml.st");

	public static String generateProfile(final String name,

			final String type,

			final WorkflowStartModeEnum startMode,

			final Map<String, String> params,

			final Resource confTemplate)

			throws IOException {

		final StringTemplate conf = new StringTemplate(IOUtils.toString(confTemplate.getInputStream()));

		conf.setAttribute("params", Maps.transformValues(params, new Function<String, String>() {

			@Override

			public String apply(final String s) {

				return StringEscapeUtils.escapeXml(s);

			}

		}));

		final StringTemplate profile = new StringTemplate(IOUtils.toString(wfTemplate.getInputStream()));

		profile.setAttribute("name", name);

		profile.setAttribute("type", type);

		profile.setAttribute("priority", WorkflowsConstants.DEFAULT_WF_PRIORITY);

		profile.setAttribute("conf", conf.toString());

		profile.setAttribute("startMode", startMode);

		return profile.toString();

	}

}

package eu.dnetlib.msro.workflows.nodes.transform;

import java.util.HashMap;

import java.util.Map;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.springframework.beans.factory.annotation.Required;

import org.springframework.core.io.ClassPathResource;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.resultset.XSLTMappedResultSetFactory;

import eu.dnetlib.enabling.resultset.client.utils.EPRUtils;

import eu.dnetlib.msro.rmi.MSROException;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

public class ApplyXsltJobNode extends SimpleJobNode {

	private String inputEprParam;

	private String outputEprParam;

	private String xsltClasspath;

	private XSLTMappedResultSetFactory xsltMappedResultSetFactory;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final String inputEpr = token.getEnv().getAttribute(inputEprParam);

		if ((inputEpr == null) || inputEpr.isEmpty()) throw new MSROException("InputEprParam (" + inputEprParam + ") not found in ENV");

		final Map<String, String> xsltParams = new HashMap<String, String>();

		for (String name : token.getFullEnv().getAttributeNames()) {

			xsltParams.put(name.replaceAll(":", "_"), token.getFullEnv().getAttribute(name));

		}

		for (String name : token.getEnv().getAttributeNames()) {

			xsltParams.put(name.replaceAll(":", "_"), token.getEnv().getAttribute(name));

		}

		xsltParams.putAll(parseJsonParameters(token));

		final W3CEndpointReference epr = xsltMappedResultSetFactory.createMappedResultSet(new EPRUtils().getEpr(inputEpr),

				(new ClassPathResource(xsltClasspath)), xsltParams);

		token.getEnv().setAttribute(outputEprParam, epr.toString());

		return Arc.DEFAULT_ARC;

	}

	public String getInputEprParam() {

		return inputEprParam;

	}

	public void setInputEprParam(final String inputEprParam) {

		this.inputEprParam = inputEprParam;

	}

	public String getOutputEprParam() {

		return outputEprParam;

	}

	public void setOutputEprParam(final String outputEprParam) {

		this.outputEprParam = outputEprParam;

	}

	public String getXsltClasspath() {

		return xsltClasspath;

	}

	public void setXsltClasspath(final String xsltClasspath) {

		this.xsltClasspath = xsltClasspath;

	}

	public XSLTMappedResultSetFactory getXsltMappedResultSetFactory() {

		return xsltMappedResultSetFactory;

	}

	@Required

	public void setXsltMappedResultSetFactory(final XSLTMappedResultSetFactory xsltMappedResultSetFactory) {

		this.xsltMappedResultSetFactory = xsltMappedResultSetFactory;

	}

}

package eu.dnetlib.msro.workflows.metawf;

public enum WorkflowStartModeEnum {

	auto, manual, disabled

}

package eu.dnetlib.msro.workflows.nodes.index;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

public class PrepareCreateIndexJobNode extends SimpleJobNode {

	private static final Log log = LogFactory.getLog(PrepareCreateIndexJobNode.class);

	private String layout;

	private String format;

	private String interpretation;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		log.info("Preparing env for CreateIndexJobNode");

		token.getEnv().setAttribute("layout", layout);

		token.getEnv().setAttribute("format", format);

		token.getEnv().setAttribute("interpretation", interpretation);

		return null;

	}

	public String getLayout() {

		return layout;

	}

	public void setLayout(final String layout) {

		this.layout = layout;

	}

	public String getFormat() {

		return format;

	}

	public void setFormat(final String format) {

		this.format = format;

	}

	public String getInterpretation() {

		return interpretation;

	}

	public void setInterpretation(final String interpretation) {

		this.interpretation = interpretation;

	}

}

package eu.dnetlib.msro.workflows.metawf;

import java.io.IOException;

import java.io.StringWriter;

import java.util.List;

import java.util.Map;

import javax.annotation.Resource;

import org.apache.commons.lang.StringEscapeUtils;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

public class WorkflowTree {

	private String id;

	private String name;

	private WorkflowStartModeEnum start = WorkflowStartModeEnum.auto;

	private List<WorkflowTree> children;

	private org.springframework.core.io.Resource template;

	@Resource

	private UniqueServiceLocator serviceLocator;

	public void populateMetaWfXml(final StringWriter sw) {

		sw.append("<WORKFLOW id='" + StringEscapeUtils.escapeXml(id) + "' name='" + StringEscapeUtils.escapeXml(name) + "'");

		if (children == null || children.isEmpty()) {

			sw.append(" />");

		} else {

			sw.append(">");

			for (WorkflowTree child : children) {

				child.populateMetaWfXml(sw);

			}

			sw.append("</WORKFLOW>");

		}

	}

	public int registerAllWorkflows(final Map<String, String> params) throws ISRegistryException, IOException {

		int count = 0;

		if (this.id == null || this.id.isEmpty()) {

			final String profile = WorkflowProfileCreator.generateProfile(name, "aggregator", start, params, template);

			this.id = serviceLocator.getService(ISRegistryService.class).registerProfile(profile);

			count++;

		}

		if (children != null) {

			for (WorkflowTree child : children) {

				count += child.registerAllWorkflows(params);

			}

		}

		return count;

	}

	public String getId() {

		return id;

	}

	public List<WorkflowTree> getChildren() {

		return children;

	}

	public void setChildren(final List<WorkflowTree> children) {

		this.children = children;

	}

	public String getName() {

		return name;

	}

	@Required

	public void setName(final String name) {

		this.name = name;

	}

	public org.springframework.core.io.Resource getTemplate() {

		return template;

	}

	@Required

	public void setTemplate(final org.springframework.core.io.Resource template) {

		this.template = template;

	}

	public WorkflowStartModeEnum getStart() {

		return start;

	}

	public void setStart(final WorkflowStartModeEnum start) {

		this.start = start;

	}

}

package eu.dnetlib.msro.workflows.nodes.index;

import java.io.IOException;

import java.io.StringReader;

import java.io.StringWriter;

import javax.annotation.Resource;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerConfigurationException;

import javax.xml.transform.TransformerException;

import javax.xml.transform.TransformerFactory;

import javax.xml.transform.dom.DOMResult;

import javax.xml.transform.dom.DOMSource;

import javax.xml.transform.stream.StreamResult;

import javax.xml.transform.stream.StreamSource;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.apache.commons.codec.binary.Base64;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.data.provision.index.rmi.IndexService;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpDocumentNotFoundException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.enabling.resultset.XSLTMappedResultSetFactory;

import eu.dnetlib.enabling.resultset.client.utils.EPRUtils;

import eu.dnetlib.enabling.resultset.rmi.ResultSetException;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;

import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;

import eu.dnetlib.msro.workflows.resultset.ProcessCountingResultSetFactory;

import eu.dnetlib.msro.workflows.util.ProgressProvider;

import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider;

public class UpdateIndexJobNode extends BlackboardJobNode implements ProgressJobNode {

	private static final Log log = LogFactory.getLog(UpdateIndexJobNode.class);

	private String eprParam;

	private String indexId;

	private String format;

	private String layout;

	private String feedingType;

	private String defaultIndexId;

	/**

	 * xslt mapped resultset factory.

	 */

	private XSLTMappedResultSetFactory xsltRSFactory;

	private ProcessCountingResultSetFactory processCountingResultSetFactory;

	private ResultsetProgressProvider progressProvider;

	/**

	 * Stylesheet which transforms a layout to another stylesheet which converts a input record to a index record.

	 */

	private org.springframework.core.io.Resource layoutToRecordStylesheet;

	/**

	 * service locator.

	 */

	@Resource

	private UniqueServiceLocator serviceLocator;

	@Override

	protected String obtainServiceId(final NodeToken token) {

		return getServiceLocator().getServiceId(IndexService.class);

	}

	@Override

	protected void prepareJob(final BlackboardJob job, final NodeToken token) throws ResultSetException, ISLookUpException, IOException, TransformerException {

		log.info("preparing blackboard job update index: " + getIndexId());

		final W3CEndpointReference epr = new EPRUtils().getEpr(token.getEnv().getAttribute(getEprParam()));

		final W3CEndpointReference mappedEpr = prepareForIndexing(epr, getFormat(), getLayout());

		progressProvider = processCountingResultSetFactory.createProgressProvider(token.getProcess(), mappedEpr);

		job.setAction("FEED");

		job.getParameters().put("resultset_epr", encode(progressProvider.getEpr().toString()));

		job.getParameters().put("id", getIndexId());

		job.getParameters().put("feeding_type", getFeedingType());

		job.getParameters().put("backend_Id", defaultIndexId);

	}

	// helpers

	/**

	 * Transforms each mdstore record into a index record.

	 * 

	 * @param mdStoreRsetEpr

	 *            mdstore resulsetset

	 * @param layout

	 *            layout

	 * @param format

	 *            format

	 * @return resultset with transformed records

	 * @throws ISLookUpException

	 *             could happen

	 * @throws IOException

	 *             could happen

	 * @throws TransformerException

	 *             could happen

	 */

	protected W3CEndpointReference prepareForIndexing(final W3CEndpointReference mdStoreRsetEpr, final String format, final String layout)

			throws ISLookUpException, IOException, TransformerException {

		final TransformerFactory factory = TransformerFactory.newInstance();

		final Transformer layoutTransformer = factory.newTransformer(new StreamSource(getLayoutToRecordStylesheet().getInputStream()));

		final DOMResult layoutToXsltXslt = new DOMResult();

		layoutTransformer.setParameter("format", format);

		layoutTransformer.transform(new StreamSource(new StringReader(getLayoutSource(format, layout))), layoutToXsltXslt);

		dumpXslt(factory, layoutToXsltXslt);

		return getXsltRSFactory().createMappedResultSet(mdStoreRsetEpr, new DOMSource(layoutToXsltXslt.getNode()),

				"dynamic layout xslt for " + format + ", " + layout);

	}

	private String getLayoutSource(final String format, final String layout) throws ISLookUpDocumentNotFoundException, ISLookUpException {

		return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(

				"collection('')//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'MDFormatDSResourceType' and .//NAME='" + format + "']//LAYOUT[@name='" + layout

						+ "']");

	}

	private void dumpXslt(final TransformerFactory factory, final DOMResult layoutToXsltXslt) throws TransformerConfigurationException, TransformerException {

		if (log.isDebugEnabled()) {

			final StringWriter buffer = new StringWriter();

			factory.newTransformer().transform(new DOMSource(layoutToXsltXslt.getNode()), new StreamResult(buffer));

			log.debug(buffer.toString());

		}

	}

	private String encode(final String epr) {

		return new String(Base64.encodeBase64(epr.getBytes()));

	}

	// setters and getters

	public String getIndexId() {

		return indexId;

	}

	public void setIndexId(final String indexId) {

		this.indexId = indexId;

	}

	public String getEprParam() {

		return eprParam;

	}

	public void setEprParam(final String eprParam) {

		this.eprParam = eprParam;

	}

	public String getFeedingType() {

		return feedingType;

	}

	public void setFeedingType(final String feedingType) {

		this.feedingType = feedingType;

	}

	public ProcessCountingResultSetFactory getProcessCountingResultSetFactory() {

		return processCountingResultSetFactory;

	}

	@Required

	public void setProcessCountingResultSetFactory(final ProcessCountingResultSetFactory processCountingResultSetFactory) {

		this.processCountingResultSetFactory = processCountingResultSetFactory;

	}

	@Override

	public ProgressProvider getProgressProvider() {

		return progressProvider;

	}

	public org.springframework.core.io.Resource getLayoutToRecordStylesheet() {

		return layoutToRecordStylesheet;

	}

	@Required

	public void setLayoutToRecordStylesheet(final org.springframework.core.io.Resource layoutToRecordStylesheet) {

		this.layoutToRecordStylesheet = layoutToRecordStylesheet;

	}

	public String getFormat() {

		return format;

	}

	public void setFormat(final String format) {

		this.format = format;

	}

	public String getLayout() {

		return layout;

	}

	public void setLayout(final String layout) {

		this.layout = layout;

	}

	public XSLTMappedResultSetFactory getXsltRSFactory() {

		return xsltRSFactory;

	}

	@Required

	public void setXsltRSFactory(final XSLTMappedResultSetFactory xsltRSFactory) {

		this.xsltRSFactory = xsltRSFactory;

	}

	/**

	 * @return the defaultIndexId

	 */

	public String getDefaultIndexId() {

		return defaultIndexId;

	}

	/**

	 * @param defaultIndexId

	 *            the defaultIndexId to set

	 */

	@Required

	public void setDefaultIndexId(final String defaultIndexId) {

		this.defaultIndexId = defaultIndexId;

	}

}

package eu.dnetlib.msro.workflows.metawf;

import java.io.IOException;

import java.io.StringWriter;

import java.util.Map;

import org.springframework.beans.factory.annotation.Required;

import org.springframework.core.io.Resource;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

public class DatasourceMetaWorkflow {

	private WorkflowTree tree;

	private Resource destroyWorkflowTemplate;

	@javax.annotation.Resource

	private UniqueServiceLocator serviceLocator;

	public int registerAllWorkflows(final Map<String, String> params) throws ISRegistryException, IOException {

		return tree.registerAllWorkflows(params);

	}

	public String registerDestroyWorkflow(final Map<String, String> params) throws ISRegistryException, IOException {

		final String profile = WorkflowProfileCreator.generateProfile("Repo BYE", "REPO_BYE", WorkflowStartModeEnum.auto, params, destroyWorkflowTemplate);

		return serviceLocator.getService(ISRegistryService.class).registerProfile(profile);

	}

	public String asXML() {

		final StringWriter sw = new StringWriter();

		tree.populateMetaWfXml(sw);

		return sw.toString();

	}

	public WorkflowTree getTree() {

		return tree;

	}

	@Required

	public void setTree(final WorkflowTree tree) {

		this.tree = tree;

	}

	public Resource getDestroyWorkflowTemplate() {

		return destroyWorkflowTemplate;

	}

	@Required

	public void setDestroyWorkflowTemplate(final Resource destroyWorkflowTemplate) {

		this.destroyWorkflowTemplate = destroyWorkflowTemplate;

	}

}

package eu.dnetlib.msro.workflows.nodes.info;

import java.io.StringReader;

import java.util.Map;

import javax.annotation.Resource;

import org.dom4j.Document;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import com.google.common.collect.Maps;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.datasources.rmi.DatasourceManagerService;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

public class MDStoreToApiExtraFieldJobNode extends SimpleJobNode {

	private String mdId;

	private String datasourceId;

	private String datasourceInterface;

	private String extraFieldForTotal;

	private String extraFieldForDate;

	private String extraFieldForMdId;

	@Resource

	private UniqueServiceLocator serviceLocator;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final String xq = "for $x in collection('/db/DRIVER/MDStoreDSResources/MDStoreDSResourceType') " +

				"where $x//RESOURCE_IDENTIFIER/@value='" + mdId + "' " +

				"return concat($x//NUMBER_OF_RECORDS, ' @=@ ', $x//LAST_STORAGE_DATE)";

		final String s = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xq);

		final String[] arr = s.split(" @=@ ");

		final Map<String, String> map = getCurrentExtraFields(datasourceId, datasourceInterface);

		map.put(extraFieldForTotal, arr[0].trim());

		map.put(extraFieldForDate, arr[1].trim());

		map.put(extraFieldForMdId, mdId);

		serviceLocator.getService(DatasourceManagerService.class).bulkUpdateApiExtraFields(datasourceId, datasourceInterface, map);

		return Arc.DEFAULT_ARC;

	}

	private Map<String, String> getCurrentExtraFields(final String repoId, final String ifaceId) throws Exception {

		final Map<String, String> res = Maps.newHashMap();

		final String profile = serviceLocator.getService(ISLookUpService.class).getResourceProfile(repoId);

		final SAXReader reader = new SAXReader();

		final Document doc = reader.read(new StringReader(profile));

		final Node ifcNode = doc.selectSingleNode("//INTERFACE[@id='" + ifaceId + "']");

		if (ifcNode != null) {

			for (Object o : ifcNode.selectNodes("./INTERFACE_EXTRA_FIELD")) {

				res.put(((Node) o).valueOf("@name"), ((Node) o).getText());

			}

		}

		return res;

	}

	public String getMdId() {

		return mdId;

	}

	public void setMdId(final String mdId) {

		this.mdId = mdId;

	}

	public String getDatasourceId() {

		return datasourceId;

	}

	public void setDatasourceId(final String datasourceId) {

		this.datasourceId = datasourceId;

	}

	public String getDatasourceInterface() {

		return datasourceInterface;

	}

	public void setDatasourceInterface(final String datasourceInterface) {

		this.datasourceInterface = datasourceInterface;

	}

	public String getExtraFieldForTotal() {

		return extraFieldForTotal;

	}

	public void setExtraFieldForTotal(final String extraFieldForTotal) {

		this.extraFieldForTotal = extraFieldForTotal;

	}