Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/BackupStatsCacheJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/BackupStatsCacheJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/BackupStatsCacheJobNode.java	(revision 43444)
@@ -0,0 +1,31 @@
+package eu.dnetlib.msro.workflows.nodes.stats;
+
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.procs.Token;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Ask the StatsManagerService to backup the stats cache of the portal specified by the additional BB parameter
+ * <code>StatsManagerServiceBBAction.BACKUP_CACHE.getTargetPortalParamName()</code>. The cache can be restored with the last generated
+ * backup by sending a "restore" BB message.
+ *
+ * @author alessia
+ * @see RestoreStatsCacheJobNode
+ */
+public class BackupStatsCacheJobNode extends AbstractStatsJobNode {
+
+	private static final Log log = LogFactory.getLog(BackupStatsCacheJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		job.setAction(StatsManagerServiceBBAction.BACKUP_CACHE.action());
+		final String portal = getPortalName(token.getEnv());
+		if (StringUtils.isNotBlank(portal)) {
+			job.getParameters().put(StatsManagerServiceBBAction.BACKUP_CACHE.getTargetPortalParamName(), portal);
+		} else {
+			log.warn(StatsManagerServiceBBAction.BACKUP_CACHE.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");
+		}
+	}
+}
Index: webapps/dnet-wds-container/trunk/src/main/resources/eu/dnetlib/cnr-site.properties
===================================================================
--- webapps/dnet-wds-container/trunk/src/main/resources/eu/dnetlib/cnr-site.properties	(revision 43443)
+++ webapps/dnet-wds-container/trunk/src/main/resources/eu/dnetlib/cnr-site.properties	(revision 43444)
@@ -32,6 +32,8 @@
 
 hbase.mapred.datatable 							=	db_openaireplus
 hbase.actions.table 							=	db_actions
+dnet.datasourcemanager.db.name=dnet_wds
+services.datasourceManager.core=dnetPostgresDSManagerCore
 
 # Mail configuration
 
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/StatsManagerServiceBBAction.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/StatsManagerServiceBBAction.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/StatsManagerServiceBBAction.java	(revision 43444)
@@ -0,0 +1,68 @@
+package eu.dnetlib.msro.workflows.nodes.stats;
+
+public enum StatsManagerServiceBBAction {
+	PROMOTE_SHADOW_STATS {
+		@Override
+		public String action() {
+			return "promoteShadow";
+		}
+	},
+	PROMOTE_SHADOW_CACHE {
+		@Override
+		public String action() {
+			return "promoteCache";
+		}
+	},
+	REFRESH_SHADOW_CACHE {
+		@Override
+		public String action() {
+			return "refreshCache";
+		}
+
+	},
+	VALIDATE_SHADOW_STATS {
+		@Override
+		public String action() {
+			return "validate";
+		}
+
+	},
+	BACKUP_CACHE {
+		@Override
+		public String action() {
+			return "backup";
+		}
+
+	},
+	RESTORE_CACHE {
+		@Override
+		public String action() {
+			return "restore";
+		}
+
+	},
+	MIGRATE_CACHE {
+		@Override
+		public String action() {
+			return "migrate";
+		}
+
+		@Override
+		public String getTargetPortalParamName() {
+			return "targetCache";
+		}
+
+	};
+
+	private static String DEFAULT_PORTAL_PARAM_NAME = "cache";
+
+	public abstract String action();
+
+	public String getSourcePortalParamName() {
+		return DEFAULT_PORTAL_PARAM_NAME;
+	}
+
+	public String getTargetPortalParamName() {
+		return DEFAULT_PORTAL_PARAM_NAME;
+	}
+}
Index: webapps/dnet-wds-container/trunk/pom.xml
===================================================================
--- webapps/dnet-wds-container/trunk/pom.xml	(revision 43443)
+++ webapps/dnet-wds-container/trunk/pom.xml	(revision 43444)
@@ -90,11 +90,11 @@
 			<version>1.0.0-SNAPSHOT</version>
 		</dependency>
 
-                <dependency>
-                        <groupId>eu.dnetlib</groupId>
-                        <artifactId>dnet-openaire</artifactId>
-                        <version>1.0.0-SNAPSHOT</version>
-                </dependency>
+		<dependency>
+			<groupId>eu.dnetlib</groupId>
+			<artifactId>dnet-wds</artifactId>
+			<version>1.0.0-SNAPSHOT</version>
+		</dependency>
 
 		<dependency>
 			<groupId>xerces</groupId>
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/utils/CleaningXsltFunctions.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/utils/CleaningXsltFunctions.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/utils/CleaningXsltFunctions.java	(revision 43444)
@@ -0,0 +1,17 @@
+package eu.dnetlib.msro.workflows.nodes.utils;
+
+import java.text.Normalizer;
+
+/**
+ * Created by claudio on 14/06/16.
+ */
+public class CleaningXsltFunctions {
+
+	public static String clean(final String s) {
+		return Normalizer.normalize(s, Normalizer.Form.NFD)
+				.replaceAll("\\(.+\\)", "")
+				.replaceAll("(\\W|\\p{InCombiningDiacriticalMarks}|\\p{Punct}|\\n|\\s)+", "")
+				.toLowerCase()
+				.trim();
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/PrepareConfiguredActionSetJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/PrepareConfiguredActionSetJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/PrepareConfiguredActionSetJobNode.java	(revision 43444)
@@ -0,0 +1,148 @@
+package eu.dnetlib.msro.workflows.nodes.dedup;
+
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.gson.Gson;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.data.hadoop.actionmanager.RawSet;
+import eu.dnetlib.rmi.enabling.ISLookUpDocumentNotFoundException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+/**
+ * The Class PrepareConfiguredActionSetJobNode.
+ */
+public class PrepareConfiguredActionSetJobNode extends SimpleJobNode {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(PrepareConfiguredActionSetJobNode.class);
+
+	/**
+	 * The dedup config sequence param.
+	 */
+	private String dedupConfigSequence;
+
+	/**
+	 * The job property.
+	 */
+	private String jobProperty;
+
+	/**
+	 * The action set path param name.
+	 */
+	private String actionSetPathParam;
+
+	private String setsParam;
+
+	/**
+	 * The service locator.
+	 */
+	@Autowired
+	private UniqueServiceLocator serviceLocator;
+
+	/*
+	 * (non-Javadoc)
+	 *
+	 * @see eu.dnetlib.msro.workflows.nodes.SimpleJobNode#execute(com.googlecode.sarasvati.NodeToken)
+	 */
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		final List<Map<String, String>> setList = Lists.newArrayList();
+
+		final Map<String, String> set = Maps.newHashMap();
+
+		final String actionSetId = getDedupConfigSequence();
+		final ISLookUpService isLookUpService = serviceLocator.getService(ISLookUpService.class);
+		final String basePath = isLookUpService.getResourceProfileByQuery(
+				"/RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value='ActionManagerServiceResourceType']//SERVICE_PROPERTIES/PROPERTY[@key='basePath']/@value/string()");
+		if (StringUtils.isBlank(basePath)) {
+			throw new IllegalStateException("missing basePath in ActionManagerService");
+		}
+
+		String actionSetDirectory;
+		try {
+			actionSetDirectory = isLookUpService.getResourceProfileByQuery(
+					"/RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value='ActionManagerSetDSResourceType' and .//SET/@id = '" + actionSetId
+							+ "']//SET/@ directory/string()");
+		} catch (ISLookUpDocumentNotFoundException e) {
+			throw new MSROException("missing directory in ActionSet profile: " + actionSetId);
+		}
+
+		final String rawSetId = RawSet.newInstance().getId();
+		set.put("rawset", rawSetId);
+		set.put("creationDate", DateUtils.now_ISO8601());
+		set.put("set", actionSetId);
+		set.put("enabled", "true");
+		set.put("jobProperty", getJobProperty());
+
+		env.setAttribute(set.get("jobProperty"), set.get("rawset"));
+
+		final String path = basePath + "/" + actionSetDirectory + "/" + rawSetId;
+		log.info("using action set path: " + path);
+		env.setAttribute(getActionSetPathParam(), path);
+
+		setList.add(set);
+		final String sets = new Gson().toJson(setList);
+		log.debug("built set: " + sets);
+
+		env.setAttribute(getSetsParam(), sets);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	/**
+	 * Gets the job property.
+	 *
+	 * @return the job property
+	 */
+	public String getJobProperty() {
+		return jobProperty;
+	}
+
+	/**
+	 * Sets the job property.
+	 *
+	 * @param jobProperty the new job property
+	 */
+	public void setJobProperty(final String jobProperty) {
+		this.jobProperty = jobProperty;
+	}
+
+	public String getActionSetPathParam() {
+		return actionSetPathParam;
+	}
+
+	public void setActionSetPathParam(final String actionSetPathParam) {
+		this.actionSetPathParam = actionSetPathParam;
+	}
+
+	public String getDedupConfigSequence() {
+		return dedupConfigSequence;
+	}
+
+	public void setDedupConfigSequence(final String dedupConfigSequence) {
+		this.dedupConfigSequence = dedupConfigSequence;
+	}
+
+	public String getSetsParam() {
+		return setsParam;
+	}
+
+	public void setSetsParam(final String setsParam) {
+		this.setsParam = setsParam;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/utils/DedupConfigurationOrchestration.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/utils/DedupConfigurationOrchestration.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/utils/DedupConfigurationOrchestration.java	(revision 43444)
@@ -0,0 +1,105 @@
+package eu.dnetlib.msro.workflows.nodes.dedup.utils;
+
+import java.util.Queue;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import eu.dnetlib.pace.config.DedupConfig;
+
+/**
+ * The Class DedupConfigurationOrchestration.
+ */
+public class DedupConfigurationOrchestration {
+
+	/**
+	 * The entity.
+	 */
+	private Entity entity;
+
+	/**
+	 * The action set id.
+	 */
+	private String actionSetId;
+
+	/**
+	 * The configurations.
+	 */
+	private Queue<DedupConfig> configurations;
+
+	public DedupConfigurationOrchestration() {
+	}
+
+	/**
+	 * Instantiates a new dedup configuration orchestration.
+	 *
+	 * @param entity         the entity
+	 * @param actionSetId    the action set id
+	 * @param configurations the configurations
+	 */
+	public DedupConfigurationOrchestration(final Entity entity, final String actionSetId, final Queue<DedupConfig> configurations) {
+		super();
+		this.setEntity(entity);
+		this.setActionSetId(actionSetId);
+		this.setConfigurations(configurations);
+	}
+
+	/**
+	 * From json.
+	 *
+	 * @param json the json
+	 * @return the dedup configuration orchestration
+	 */
+	public static DedupConfigurationOrchestration fromJSON(final String json) {
+		return new Gson().fromJson(json, DedupConfigurationOrchestration.class);
+	}
+
+	/**
+	 * Gets the entity.
+	 *
+	 * @return the entity
+	 */
+	public Entity getEntity() {
+		return entity;
+	}
+
+	public void setEntity(final Entity entity) {
+		this.entity = entity;
+	}
+
+	/**
+	 * Gets the action set id.
+	 *
+	 * @return the action set id
+	 */
+	public String getActionSetId() {
+		return actionSetId;
+	}
+
+	public void setActionSetId(final String actionSetId) {
+		this.actionSetId = actionSetId;
+	}
+
+	/**
+	 * Gets the configurations.
+	 *
+	 * @return the configurations
+	 */
+	public Queue<DedupConfig> getConfigurations() {
+		return configurations;
+	}
+
+	public void setConfigurations(final Queue<DedupConfig> configurations) {
+		this.configurations = configurations;
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see java.lang.Object#toString()
+	 */
+	@Override
+	public String toString() {
+		return new GsonBuilder().setPrettyPrinting().create().toJson(this);
+	}
+
+}
Index: modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/actionmanager/is/ISClient.java
===================================================================
--- modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/actionmanager/is/ISClient.java	(revision 43443)
+++ modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/actionmanager/is/ISClient.java	(revision 43444)
@@ -10,7 +10,6 @@
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
-
 import eu.dnetlib.enabling.locators.UniqueServiceLocator;
 import eu.dnetlib.miscutils.datetime.DateUtils;
 import eu.dnetlib.rmi.data.hadoop.actionmanager.ActionManagerException;
@@ -21,7 +20,6 @@
 import eu.dnetlib.rmi.enabling.ISLookUpService;
 import eu.dnetlib.rmi.enabling.ISRegistryException;
 import eu.dnetlib.rmi.enabling.ISRegistryService;
-import eu.dnetlib.soap.EndpointReferenceBuilder;
 import org.antlr.stringtemplate.StringTemplate;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -34,6 +32,10 @@
 public class ISClient {
 
 	private static final Log log = LogFactory.getLog(ISClient.class); // NOPMD by marko on 11/24/08 5:02 PM
+	/**
+	 * endpoint builder.
+	 */
+	private static final String ENDPOINT_TEMPLATE = "http://%s:%s/%s/services/actionManager";
 	@Resource
 	private UniqueServiceLocator serviceLocator;
 	/**
@@ -44,16 +46,15 @@
 	 * service endpoint.
 	 */
 	private Endpoint endpoint;
-	/**
-	 * endpoint builder.
-	 */
-	private EndpointReferenceBuilder<Endpoint> eprBuilder;
+	private String hostname;
+	private String port;
+	private String context;
 
 	public String registerSetProfile(final ActionManagerSet set) throws ActionManagerException {
 		if (existsSet(set.getId())) { throw new ActionManagerException("Set " + set.getId() + " already registered"); }
 		try {
 			StringTemplate template = new StringTemplate(actionManagerSetDsTemplate.getTemplate());
-			template.setAttribute("serviceUri", eprBuilder.getAddress(endpoint));
+			template.setAttribute("serviceUri", String.format(ENDPOINT_TEMPLATE, hostname, port, context));
 			template.setAttribute("set", set);
 			template.setAttribute("latest", RawSet.newInstance());
 			return serviceLocator.getService(ISRegistryService.class).registerProfile(template.toString());
@@ -284,13 +285,30 @@
 		this.endpoint = endpoint;
 	}
 
-	public EndpointReferenceBuilder<Endpoint> getEprBuilder() {
-		return eprBuilder;
+	public String getHostname() {
+		return hostname;
 	}
 
 	@Required
-	public void setEprBuilder(final EndpointReferenceBuilder<Endpoint> eprBuilder) {
-		this.eprBuilder = eprBuilder;
+	public void setHostname(final String hostname) {
+		this.hostname = hostname;
 	}
 
+	public String getPort() {
+		return port;
+	}
+
+	@Required
+	public void setPort(final String port) {
+		this.port = port;
+	}
+
+	public String getContext() {
+		return context;
+	}
+
+	@Required
+	public void setContext(final String context) {
+		this.context = context;
+	}
 }
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/DedupDuplicateScanJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/DedupDuplicateScanJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/DedupDuplicateScanJobNode.java	(revision 43444)
@@ -0,0 +1,48 @@
+package eu.dnetlib.msro.workflows.nodes.dedup;
+
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.blackboard.BlackboardWorkflowJobListener;
+import eu.dnetlib.msro.workflows.nodes.dedup.utils.DedupConfigurationOrchestration;
+import eu.dnetlib.msro.workflows.procs.Token;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class DedupDuplicateScanJobNode extends DedupConfigurationAwareJobNode {
+
+	private static final Log log = LogFactory.getLog(DedupDuplicateScanJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	@Override
+	protected BlackboardWorkflowJobListener generateBlackboardListener(final Token token) {
+		return new DedupBlackboardWorkflowJobListener(token);
+	}
+
+	private class DedupBlackboardWorkflowJobListener extends BlackboardWorkflowJobListener {
+
+		public DedupBlackboardWorkflowJobListener(final Token token) {
+			super(token);
+		}
+
+		@Override
+		protected void onDone(final BlackboardJob job) {
+
+			final DedupConfigurationOrchestration confs = getDedupConfigurationOrchestration();
+
+			confs.getConfigurations().poll();
+
+			log.info("checking dedup configs queue, size: " + confs.getConfigurations().size());
+
+			if (CollectionUtils.isEmpty(confs.getConfigurations())) {
+				log.info("dedup similarity scan done");
+
+				getToken().release("done");
+			} else {
+				log.debug("remaining confs: " + confs);
+
+				getToken().release(Arc.DEFAULT_ARC);
+			}
+		}
+	}
+
+}
Index: modules/dnet-hadoop-services/trunk/src/main/resources/eu/dnetlib/data/actionmanager/applicationContext-dnet-actionmanager-service.xml
===================================================================
--- modules/dnet-hadoop-services/trunk/src/main/resources/eu/dnetlib/data/actionmanager/applicationContext-dnet-actionmanager-service.xml	(revision 43443)
+++ modules/dnet-hadoop-services/trunk/src/main/resources/eu/dnetlib/data/actionmanager/applicationContext-dnet-actionmanager-service.xml	(revision 43444)
@@ -1,17 +1,14 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-       xmlns:jaxws="http://cxf.apache.org/jaxws" xmlns:sec="http://cxf.apache.org/configuration/security"
-       xmlns:wsa="http://cxf.apache.org/ws/addressing" xmlns:p="http://www.springframework.org/schema/p"
-       xmlns:http="http://cxf.apache.org/transports/http/configuration" xmlns:t="http://dnetlib.eu/springbeans/t"
-       xmlns:template="http://dnetlib.eu/springbeans/template" xmlns:util="http://www.springframework.org/schema/util"
+       xmlns:jaxws="http://cxf.apache.org/jaxws"
+       xmlns:p="http://www.springframework.org/schema/p"
+       xmlns:http="http://cxf.apache.org/transports/http/configuration"
        xmlns="http://www.springframework.org/schema/beans"
        xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
-                                    http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd
-                                    http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd
+
+
                                     http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd
-                            http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd
-                            http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd
-                            http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">
+                            http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd">
 
 	<bean id="actionManagerService" class="eu.dnetlib.data.actionmanager.ActionManagerServiceImpl"
 	      p:notificationHandler-ref="actionManagerNotificationHandler"
@@ -69,7 +66,9 @@
 	<!-- Information Service Client -->
 	<bean id="actionmanagerISClient" class="eu.dnetlib.data.actionmanager.is.ISClient"
 	      p:actionManagerSetDsTemplate-ref="actionManagerSetDsTemplate"
-	      p:endpoint-ref="actionManagerServiceEndpoint" p:eprBuilder-ref="jaxwsEndpointReferenceBuilder" />
+	      p:endpoint-ref="actionManagerServiceEndpoint" p:hostname="${container.hostname}"
+	      p:context="${container.context}"
+	      p:port="${container.port}"/>
 
 	<bean id="actionManagerSetDsTemplate"
 	      class="eu.dnetlib.springutils.stringtemplate.StringTemplateFactory"
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/BuildSimilarityMeshJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/BuildSimilarityMeshJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/BuildSimilarityMeshJobNode.java	(revision 43444)
@@ -0,0 +1,159 @@
+package eu.dnetlib.msro.workflows.nodes.dedup;
+
+import java.io.StringReader;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Queue;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Queues;
+import eu.dnetlib.data.proto.TypeProtos.Type;
+import eu.dnetlib.enabling.resultset.client.ResultSetClient;
+import eu.dnetlib.enabling.resultset.factory.ResultSetFactory;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.AsyncJobNode;
+import eu.dnetlib.msro.workflows.nodes.utils.Similarity;
+import eu.dnetlib.msro.workflows.nodes.utils.SimilarityMeshBuilder;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.common.ResultSet;
+import org.antlr.stringtemplate.StringTemplate;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.DocumentException;
+import org.dom4j.Node;
+import org.dom4j.io.SAXReader;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Required;
+
+public class BuildSimilarityMeshJobNode extends AsyncJobNode {
+
+	private static final Log log = LogFactory.getLog(BuildSimilarityMeshJobNode.class);
+
+	/**
+	 * The result set client factory.
+	 */
+	@Autowired
+	private ResultSetClient resultSetClient;
+
+	@Autowired
+	private ResultSetFactory resultSetFactory;
+
+	private StringTemplate similarity;
+
+	private String inputEprParam;
+
+	private String outputEprParam;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		final ResultSet<?> inputRs = env.getAttribute(getInputEprParam(), ResultSet.class);
+		final Iterator<String> it = resultSetClient.iter(inputRs, String.class).iterator();
+
+		final Queue<Object> queue = Queues.newLinkedBlockingQueue();
+		final SAXReader reader = new SAXReader();
+
+		if (it.hasNext()) {
+			populateQueue(queue, reader, it.next());
+		}
+
+		final ResultSet<String> rsOut = resultSetFactory.createResultSet(() -> {
+			return new Iterator<String>() {
+
+				@Override
+				public boolean hasNext() {
+					synchronized (queue) {
+						return !queue.isEmpty();
+					}
+				}
+
+				@Override
+				public String next() {
+					synchronized (queue) {
+						final Object o = queue.poll();
+						while (queue.isEmpty() && it.hasNext()) {
+							populateQueue(queue, reader, it.next());
+						}
+						return buildSimilarity((Similarity) o);
+					}
+				}
+
+				@Override
+				public void remove() {
+					throw new UnsupportedOperationException();
+				}
+			};
+		});
+
+		env.setAttribute(getOutputEprParam(), rsOut);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	private void populateQueue(final Queue<Object> q, final SAXReader r, final String xml) {
+		try {
+			final Document d = r.read(new StringReader(xml));
+			final String groupid = d.valueOf("//FIELD[@name='id']");
+			final List<?> items = d.selectNodes("//FIELD[@name='group']/ITEM");
+			final String entitytype = d.valueOf("//FIELD[@name='entitytype']");
+			final List<String> group = Lists.newArrayList();
+			for (final Object id : items) {
+				group.add(((Node) id).getText());
+			}
+			// compute the full mesh
+			final Type type = Type.valueOf(entitytype);
+
+			final List<Similarity> mesh = SimilarityMeshBuilder.build(type, group);
+			// total += mesh.size();
+			if (log.isDebugEnabled()) {
+				log.debug(String.format("built mesh for group '%s', size %d", groupid, mesh.size()));
+			}
+			for (final Similarity s : mesh) {
+				if (log.isDebugEnabled()) {
+					log.debug(String.format("adding to queue: %s", s.toString()));
+				}
+				q.add(s);
+			}
+		} catch (final DocumentException e) {
+			log.error("invalid document: " + xml);
+		}
+	}
+
+	private String buildSimilarity(final Similarity s) {
+		final StringTemplate template = new StringTemplate(getSimilarity().getTemplate());
+
+		template.setAttribute("source", s.getPair().getKey());
+		template.setAttribute("target", s.getPair().getValue());
+		template.setAttribute("type", s.getType().toString());
+
+		final String res = template.toString();
+		return res;
+	}
+
+	public String getInputEprParam() {
+		return inputEprParam;
+	}
+
+	public void setInputEprParam(final String inputEprParam) {
+		this.inputEprParam = inputEprParam;
+	}
+
+	public String getOutputEprParam() {
+		return outputEprParam;
+	}
+
+	public void setOutputEprParam(final String outputEprParam) {
+		this.outputEprParam = outputEprParam;
+	}
+
+	public StringTemplate getSimilarity() {
+		return similarity;
+	}
+
+	@Required
+	public void setSimilarity(final StringTemplate similarity) {
+		this.similarity = similarity;
+	}
+
+}
Index: modules/dnet-wds/trunk/test/java/eu/dnetlib/wds/collector/plugins/CMRIteratorTest.java
===================================================================
--- modules/dnet-wds/trunk/test/java/eu/dnetlib/wds/collector/plugins/CMRIteratorTest.java	(revision 43443)
+++ modules/dnet-wds/trunk/test/java/eu/dnetlib/wds/collector/plugins/CMRIteratorTest.java	(revision 43444)
@@ -2,6 +2,8 @@
 
 import java.util.Iterator;
 
+import org.antlr.stringtemplate.StringTemplate;
+import org.apache.commons.io.IOUtils;
 import org.junit.Test;
 import org.springframework.core.io.ClassPathResource;
 
@@ -15,20 +17,25 @@
 	@Test
 	public void test() throws Exception {
 
-		CMRCollectorPlugin plugin = new CMRCollectorPlugin();
+		ClassPathResource resource = new ClassPathResource("eu/dnetlib/wds/collector/plugins/CMR2Datasource.st");
 
-		ClassPathResource resource = new ClassPathResource("eu/dnetlib/wds/collector/plugins/CMR2Datacite.st");
+		StringTemplate st = new StringTemplate(IOUtils.toString(resource.getInputStream()));
 
-		plugin.setClassPathTemplate(resource);
-		final Iterable<String> collect = plugin.collect(null, null, null);
 
 		CMRDatasourcePlugin datasourcePlugin = new CMRDatasourcePlugin();
 
+		datasourcePlugin.setXmlTemplate(st);
+
 		final Iterator<String> iterator = datasourcePlugin.collect(null, null, null).iterator();
 
-		for (String s : datasourcePlugin.collect(null, null, null)) {
-			System.out.println("s = " + s);
-		}
+		System.out.println(iterator.next());
+		System.out.println(iterator.next());
+		System.out.println(iterator.next());
+		System.out.println(iterator.next());
+		System.out.println(iterator.next());
+
+
+
 	}
 
 }
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/profiles/RepositoryXsltFunctions.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/profiles/RepositoryXsltFunctions.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/profiles/RepositoryXsltFunctions.java	(revision 43444)
@@ -0,0 +1,136 @@
+package eu.dnetlib.msro.workflows.profiles;
+
+/**
+ * Created by claudio on 14/06/16.
+ */
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.antlr.stringtemplate.StringTemplate;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringEscapeUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+
+public class RepositoryXsltFunctions {
+
+	private static final Log log = LogFactory.getLog(RepositoryXsltFunctions.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	private static Resource ifaceTemplate = new ClassPathResource("/eu/dnetlib/msro/openaireplus/workflows/profiles/repo_interface.st");
+
+	private static Resource paramTemplate = new ClassPathResource("/eu/dnetlib/msro/openaireplus/workflows/profiles/repo_param.st");
+
+	public static String buildInterface(final String infopackage) {
+
+		final List<String> formats = Lists.newArrayList();
+		final List<String> sets = Lists.newArrayList();
+		final Map<String, String> otherParams = Maps.newHashMap();
+
+		for (String param : parseParams(getValueBetween(infopackage, "===6===", null))) {
+			StringTemplate p = getTemplate(paramTemplate);
+			String paramName = StringUtils.substringBefore(param, "=").trim();
+			String paramValue = StringUtils.substringAfter(param, "=").trim();
+
+			p.setAttribute("param", paramName.toUpperCase());
+			p.setAttribute("value", paramValue);
+
+			try {
+				switch (Params.valueOf(paramName.toUpperCase())) {
+				case FORMAT:
+					formats.add(p.toString());
+					break;
+				case SET:
+					sets.add(p.toString());
+					break;
+				}
+			} catch (Exception e) {
+				if ((paramName != null) && !paramName.isEmpty()) {
+					otherParams.put(paramName, StringEscapeUtils.escapeXml(paramValue));
+				}
+			}
+		}
+
+		final StringTemplate i = getTemplate(ifaceTemplate);
+		i.setAttribute("id", getValueBetween(infopackage, null, "===1==="));
+		i.setAttribute("fileMode", getValueBetween(infopackage, "===1===", "===2==="));
+		i.setAttribute("fileDesc", getValueBetween(infopackage, "===2===", "===3==="));
+		i.setAttribute("iisWf", getValueBetween(infopackage, "===3===", "===4==="));
+		i.setAttribute("baseUrl", getBaseUrl(getValueBetween(infopackage, "===4===", "===5===")));
+		i.setAttribute("accessProtocol", getAccessProtocol(getValueBetween(infopackage, "===5===", "===6==="), otherParams));
+		i.setAttribute("formats", ensureMinOccurs(formats, "<FORMAT/>"));
+		i.setAttribute("sets", ensureMinOccurs(sets, "<SET/>"));
+		if (!otherParams.isEmpty()) {
+			i.setAttribute("otherParams", otherParams);
+		}
+
+		final String iface = i.toString();
+		return iface;
+	}
+
+	;
+
+	private static String getValueBetween(final String s, final String pre, final String post) {
+		if ((pre == null) && (post == null)) {
+			return s;
+		} else if (pre == null) {
+			return StringUtils.substringBefore(s, post);
+		} else if (post == null) {
+			return StringUtils.substringAfter(s, pre);
+		} else {
+			return StringUtils.substringBetween(s, pre, post);
+		}
+	}
+
+	private static List<String> ensureMinOccurs(final List<String> list, final String empty) {
+		if (list.isEmpty()) {
+			list.add(empty);
+		}
+		return list;
+	}
+
+	private static Iterable<String> parseParams(final String s) {
+		return Splitter.on("***").omitEmptyStrings().trimResults().split(s);
+	}
+
+	private static String getBaseUrl(final String url) {
+		final StringTemplate b = getTemplate(paramTemplate);
+		b.setAttribute("param", "BASE_URL");
+		b.setAttribute("value", url);
+		return b.toString();
+	}
+
+	private static String getAccessProtocol(final String protocol, final Map<String, String> otherParams) {
+		final StringTemplate a = getTemplate(paramTemplate);
+		a.setAttribute("param", "ACCESS_PROTOCOL");
+		a.setAttribute("value", protocol);
+		if ((otherParams != null) && !otherParams.isEmpty()) {
+			a.setAttribute("attrs", otherParams);
+		}
+		return a.toString();
+	}
+
+	private static StringTemplate getTemplate(final Resource res) {
+		final StringWriter body = new StringWriter();
+		try {
+			IOUtils.copy(res.getInputStream(), body);
+			return new StringTemplate(body.toString());
+		} catch (IOException e) {
+			log.error("unable to get template", e);
+			throw new RuntimeException(e);
+		}
+	}
+
+	private enum Params {
+		FORMAT, SET
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/PatchHostedBy.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/PatchHostedBy.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/PatchHostedBy.java	(revision 43444)
@@ -0,0 +1,110 @@
+package eu.dnetlib.msro.workflows.nodes.hostedby;
+
+import java.io.StringReader;
+import java.util.Map;
+import java.util.function.UnaryOperator;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.Element;
+import org.dom4j.Node;
+import org.dom4j.io.SAXReader;
+
+/**
+ * The Class PatchHostedBy.
+ */
+public class PatchHostedBy implements UnaryOperator<String> {
+
+	/**
+	 * The Constant log.
+	 */
+	private static final Log log = LogFactory.getLog(PatchHostedBy.class);
+	/**
+	 * The set spec hosted by map.
+	 */
+	private final Map<String, HostedByEntry> setSpecHostedByMap;
+	/**
+	 * The counters.
+	 */
+	private final HostedByCounters counters;
+	/**
+	 * The xpath.
+	 */
+	private final String xpath;
+	/**
+	 * The reader.
+	 */
+	private final SAXReader reader = new SAXReader();
+
+	/**
+	 * Instantiates a new patch hosted by.
+	 *
+	 * @param setSpecHostedByMap the set spec hosted by map
+	 * @param xpath              the xpath
+	 * @param counters           the counters
+	 */
+	public PatchHostedBy(final Map<String, HostedByEntry> setSpecHostedByMap, final String xpath, final HostedByCounters counters) {
+		this.setSpecHostedByMap = setSpecHostedByMap;
+		this.xpath = xpath;
+		this.counters = counters;
+		if (log.isDebugEnabled()) {
+			log.debug("****************************************");
+			log.debug("SetSpec/hostedBy map:");
+			for (final Map.Entry<String, HostedByEntry> e : setSpecHostedByMap.entrySet()) {
+				log.debug("   " + e.getKey() + " -> " + e.getValue());
+			}
+			log.debug("****************************************");
+		}
+	}
+
+	/**
+	 * Evaluate.
+	 *
+	 * @param record the record
+	 * @return the string
+	 */
+	@Override
+	public String apply(final String record) {
+		try {
+			final Document doc = this.reader.read(new StringReader(record));
+			final Element node = (Element) doc.selectSingleNode("//*[local-name()='hostedBy']");
+			if (node != null) {
+				final HostedByEntry ds = findHostedBy(doc);
+				if (ds != null) {
+					node.addAttribute("id", ds.getId());
+					node.addAttribute("name", ds.getName());
+					this.counters.increaseCounter(ds.getId());
+				}
+				return doc.asXML();
+			} else if (log.isDebugEnabled()) {
+				log.debug(" -- Missing hostedBy --");
+			}
+		} catch (final Throwable e) {
+			log.error("Error adding hosted by to " + record);
+		}
+		return record;
+	}
+
+	/**
+	 * Find hosted by.
+	 *
+	 * @param doc the doc
+	 * @return the hosted by entry
+	 */
+	private HostedByEntry findHostedBy(final Document doc) {
+		for (final Object o : doc.selectNodes(this.xpath)) {
+			final String set = ((Node) o).getText().trim();
+			if (this.setSpecHostedByMap.containsKey(set)) {
+				if (log.isDebugEnabled()) {
+					log.debug(set + " -> " + this.setSpecHostedByMap.get(set));
+				}
+				return this.setSpecHostedByMap.get(set);
+			} else if (log.isDebugEnabled()) {
+				log.debug(set + " -> UNKNOWN REPO");
+			}
+		}
+		return null;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/PatchHostedByJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/PatchHostedByJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/PatchHostedByJobNode.java	(revision 43444)
@@ -0,0 +1,196 @@
+package eu.dnetlib.msro.workflows.nodes.hostedby;
+
+import java.io.StringReader;
+import java.util.Map;
+import java.util.function.UnaryOperator;
+
+import com.google.common.collect.Maps;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.resultset.client.ResultSetClient;
+import eu.dnetlib.enabling.resultset.factory.ResultSetFactory;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.common.ResultSet;
+import eu.dnetlib.rmi.data.DatabaseException;
+import eu.dnetlib.rmi.data.DatabaseService;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.DocumentException;
+import org.dom4j.io.SAXReader;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+
+public class PatchHostedByJobNode extends SimpleJobNode {
+
+	private static final Log log = LogFactory.getLog(PatchHostedByJobNode.class);
+	private String inputEprParam;
+	private String outputEprParam;
+	@Value("${dnet.openaire.db.name}")
+	private String dbName;
+	private String countersParam;
+	private String hostedbyMapTable;
+	private String xpathEntry;
+	private String overrideDataSourceId;
+	@Autowired
+	private UniqueServiceLocator serviceLocator;
+	@Autowired
+	private ResultSetFactory resultSetFactory;
+	@Autowired
+	private ResultSetClient resultSetClient;
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	protected String execute(final Env env) throws Exception {
+		final ResultSet<?> inputEpr = env.getAttribute(this.inputEprParam, ResultSet.class);
+		final HostedByCounters counters = new HostedByCounters();
+		String datasourceId;
+		if (StringUtils.isEmpty(this.overrideDataSourceId)) {
+			datasourceId = env.getAttribute("parentDatasourceId", String.class);
+		} else {
+			datasourceId = getOverrideDataSourceId();
+		}
+
+		final UnaryOperator<String> mapper = new PatchHostedBy(loadHostedByMap(datasourceId), getXpathEntry(), counters);
+		final ResultSet<String> epr = this.resultSetFactory.map(inputEpr, String.class, mapper);
+
+		env.setAttribute(this.outputEprParam, epr);
+		env.setAttribute(this.countersParam, counters);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	private Map<String, HostedByEntry> loadHostedByMap(final String datasourceId) throws DocumentException, DatabaseException {
+		final Map<String, HostedByEntry> map = Maps.newHashMap();
+
+		final String sql = "SELECT d.id, d.officialname, p.entry from %s p JOIN datasources d ON (p.datasourceid = d.id) WHERE p.oa_source_id= '%s'";
+
+		final ResultSet<String> epr = this.serviceLocator.getService(DatabaseService.class).searchSQL(getDbName(),
+				String.format(sql, getHostedbyMapTable(), datasourceId));
+
+		final SAXReader reader = new SAXReader();
+		for (final String s : this.resultSetClient.iter(epr, String.class)) {
+			final Document doc = reader.read(new StringReader(s));
+			final String entry = doc.valueOf("//FIELD[@name='entry']");
+			final String dsId = doc.valueOf("//FIELD[@name='id']");
+			final String dsName = doc.valueOf("//FIELD[@name='officialname']");
+			map.put(entry, new HostedByEntry(dsId, dsName));
+		}
+
+		log.info(String.format("built hostedByMap from dsId '%s', size: '%s'", datasourceId, map.size()));
+
+		return map;
+	}
+
+	/**
+	 * Getter for property 'inputEprParam'.
+	 *
+	 * @return Value for property 'inputEprParam'.
+	 */
+	public String getInputEprParam() {
+		return this.inputEprParam;
+	}
+
+	/**
+	 * Setter for property 'inputEprParam'.
+	 *
+	 * @param inputEprParam Value to set for property 'inputEprParam'.
+	 */
+	public void setInputEprParam(final String inputEprParam) {
+		this.inputEprParam = inputEprParam;
+	}
+
+	/**
+	 * Getter for property 'outputEprParam'.
+	 *
+	 * @return Value for property 'outputEprParam'.
+	 */
+	public String getOutputEprParam() {
+		return this.outputEprParam;
+	}
+
+	/**
+	 * Setter for property 'outputEprParam'.
+	 *
+	 * @param outputEprParam Value to set for property 'outputEprParam'.
+	 */
+	public void setOutputEprParam(final String outputEprParam) {
+		this.outputEprParam = outputEprParam;
+	}
+
+	/**
+	 * Getter for property 'dbName'.
+	 *
+	 * @return Value for property 'dbName'.
+	 */
+	public String getDbName() {
+		return this.dbName;
+	}
+
+	/**
+	 * Setter for property 'dbName'.
+	 *
+	 * @param dbName Value to set for property 'dbName'.
+	 */
+	public void setDbName(final String dbName) {
+		this.dbName = dbName;
+	}
+
+	/**
+	 * Getter for property 'countersParam'.
+	 *
+	 * @return Value for property 'countersParam'.
+	 */
+	public String getCountersParam() {
+		return this.countersParam;
+	}
+
+	/**
+	 * Setter for property 'countersParam'.
+	 *
+	 * @param countersParam Value to set for property 'countersParam'.
+	 */
+	public void setCountersParam(final String countersParam) {
+		this.countersParam = countersParam;
+	}
+
+	/**
+	 * @return the hostedbyMapTable
+	 */
+	public String getHostedbyMapTable() {
+		return this.hostedbyMapTable;
+	}
+
+	/**
+	 * @param hostedbyMapTable the hostedbyMapTable to set
+	 */
+	public void setHostedbyMapTable(final String hostedbyMapTable) {
+		this.hostedbyMapTable = hostedbyMapTable;
+	}
+
+	public String getXpathEntry() {
+		return this.xpathEntry;
+	}
+
+	public void setXpathEntry(final String xpathEntry) {
+		this.xpathEntry = xpathEntry;
+	}
+
+	/**
+	 * @return the overrideDataSourceId
+	 */
+	public String getOverrideDataSourceId() {
+		return this.overrideDataSourceId;
+	}
+
+	/**
+	 * @param overrideDataSourceId the overrideDataSourceId to set
+	 */
+	public void setOverrideDataSourceId(final String overrideDataSourceId) {
+		this.overrideDataSourceId = overrideDataSourceId;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/UpsertHostedByApisJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/UpsertHostedByApisJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/UpsertHostedByApisJobNode.java	(revision 43444)
@@ -0,0 +1,100 @@
+package eu.dnetlib.msro.workflows.nodes.hostedby;
+
+import java.util.Map.Entry;
+import javax.annotation.Resource;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.datasource.DatasourceDesc;
+import eu.dnetlib.rmi.datasource.DatasourceManagerService;
+import eu.dnetlib.rmi.datasource.DatasourceManagerServiceException;
+import eu.dnetlib.rmi.datasource.IfaceDesc;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class UpsertHostedByApisJobNode extends SimpleJobNode {
+
+	public static final String extraFieldsForTotal = "last_aggregation_total";
+	public static final String extraFieldForDate = "last_aggregation_date";
+	public static final String extraFieldForMdId = "last_aggregation_mdId";
+	private static final Log log = LogFactory.getLog(UpsertHostedByApisJobNode.class);
+	private static final String HOSTED_BY_COMPLIANCE = "hostedBy";
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+	private String countersParam;
+	private String mdId;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+		final HostedByCounters counters = env.getAttribute(this.countersParam, HostedByCounters.class);
+		final DatasourceManagerService dsManager = this.serviceLocator.getService(DatasourceManagerService.class);
+		final String date = DateUtils.now_ISO8601();
+		final String namespacePrefix = env.getAttribute("namespacePrefix", String.class);
+
+		log.info(counters);
+
+		for (final Entry<String, Integer> e : counters.getCounters().entrySet()) {
+			updateHostedByApi(dsManager, e.getKey(), namespacePrefix, date, e.getValue());
+		}
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	private void updateHostedByApi(final DatasourceManagerService dsManager,
+			final String dsId,
+			final String namepsacePrefix,
+			final String date,
+			final int size) {
+		log.info("Verifying hostedBy api in ds: " + dsId);
+		try {
+			final DatasourceDesc ds = dsManager.getDatasource(dsId);
+
+			for (final IfaceDesc iface : ds.getInterfaces()) {
+				if (HOSTED_BY_COMPLIANCE.equals(iface.getCompliance())) {
+
+					// dsManager.updateExtraField(dsId, iface.getId(), namepsacePrefix + ":" + extraFieldForDate, date, false);
+					// dsManager.updateExtraField(dsId, iface.getId(), namepsacePrefix + ":" + extraFieldsForTotal, Integer.toString(size),
+					// false);
+					// dsManager.updateExtraField(dsId, iface.getId(), namepsacePrefix + ":" + extraFieldForMdId, mdId, false);
+					return;
+				}
+			}
+
+			final IfaceDesc iface = new IfaceDesc();
+			iface.setId("api_________::" + dsId + "::hostedBy");
+			iface.setTypology(ds.getDatasourceClass());
+			iface.setCompliance(HOSTED_BY_COMPLIANCE);
+			iface.setAccessProtocol("UNKNOWN");
+			iface.setContentDescription("metadata");
+			iface.setBaseUrl("");
+			iface.setActive(false);
+			iface.setRemovable(true);
+			iface.getExtraFields().put(namepsacePrefix + ":" + extraFieldForDate, date);
+			iface.getExtraFields().put(namepsacePrefix + ":" + extraFieldsForTotal, Integer.toString(size));
+			iface.getExtraFields().put(namepsacePrefix + ":" + extraFieldForMdId, this.mdId);
+			dsManager.addInterface(dsId, iface);
+		} catch (final DatasourceManagerServiceException e) {
+			log.warn("Error setting hostedBy api of ds: " + dsId, e);
+		}
+	}
+
+	public String getMdId() {
+		return this.mdId;
+	}
+
+	public void setMdId(final String mdId) {
+		this.mdId = mdId;
+	}
+
+	public String getCountersParam() {
+		return this.countersParam;
+	}
+
+	public void setCountersParam(final String countersParam) {
+		this.countersParam = countersParam;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/MinDistSearchHadoopJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/MinDistSearchHadoopJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/MinDistSearchHadoopJobNode.java	(revision 43444)
@@ -0,0 +1,177 @@
+package eu.dnetlib.msro.workflows.nodes.dedup;
+
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.blackboard.BlackboardWorkflowJobListener;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.rmi.data.hadoop.HadoopService;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+/**
+ * Created by claudio on 14/10/15.
+ */
+public class MinDistSearchHadoopJobNode extends DedupConfigurationLoaderJobNode {
+
+	private static final Log log = LogFactory.getLog(MinDistSearchHadoopJobNode.class);
+
+	private final static String StatusParam = "MinDistSearchHadoopJobNode.status";
+	private final static String DepthParam = "mindist_recursion_depth";
+	private final static String UpdateCounterParam = "UpdateCounter.UPDATED";
+	private final static String DebugParam = "mindist_DEBUG";
+
+	@Autowired
+	private UniqueServiceLocator serviceLocator;
+	private boolean debug = false;
+	private String outPathParam;
+
+	private String workDir;
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+
+		String depthString = token.getEnv().getAttribute(DepthParam, String.class);
+		log.debug(String.format("found depthParam: '%s'", depthString));
+		if (StringUtils.isBlank(depthString)) {
+			depthString = "0";
+		}
+
+		int depth = Integer.valueOf(depthString);
+		final String outputPath = getPath(getWorkDir(), depth);
+
+		final HadoopService hadoopService = serviceLocator.getService(HadoopService.class);
+		switch (getStatusFromEnv(token.getEnv())) {
+
+		case DATALOAD:
+
+			setHadoopJob("dedupSimilarity2GraphJob");
+
+			job.getParameters().put("mapred.output.dir", getPath(getWorkDir(), depth) + "/out");
+
+			hadoopService.createHdfsDirectory(getCluster(), outputPath, true);
+
+			break;
+		case DEPTH_N:
+
+			setHadoopJob("dedupMinDistGraphJob");
+
+			final String newOutputPath = getPath(getWorkDir(), depth + 1);
+			hadoopService.createHdfsDirectory(getCluster(), newOutputPath, true);
+
+			job.getParameters().put(DepthParam, String.valueOf(depth));
+			job.getParameters().put(DebugParam, String.valueOf(isDebug()));
+
+			job.getParameters().put("mapred.input.dir", outputPath + "/out");
+			job.getParameters().put("mapred.output.dir", newOutputPath + "/out");
+
+			if (log.isDebugEnabled()) {
+				log.debug(String.format("input job parameters: %s", job.getParameters()));
+			}
+
+			token.getEnv().setAttribute(DepthParam, String.valueOf(depth + 1));
+			token.getEnv().setAttribute(getOutPathParam(), newOutputPath + "/out");
+
+			break;
+		}
+
+		super.prepareJob(job, token);
+	}
+
+	private String getPath(final String basePath, final int depth) {
+
+		log.info("got basePath: " + basePath);
+
+		Path fsPath = FileSystems.getDefault().getPath(basePath, "depth_" + depth);
+		final String path = fsPath.toAbsolutePath().toString();
+
+		log.info("built outputPath: " + path);
+
+		return path;
+	}
+
+	private STATUS getStatusFromEnv(final Env env) {
+		if (StringUtils.isBlank(env.getAttribute(StatusParam, String.class))) {
+			return STATUS.DATALOAD;
+		}
+		STATUS current = STATUS.DATALOAD;
+		try {
+			current = STATUS.valueOf(env.getAttribute(StatusParam, String.class));
+			log.debug("found status: " + current.toString());
+		} catch (IllegalArgumentException e) {}
+		return current;
+	}
+
+	@Override
+	protected BlackboardWorkflowJobListener generateBlackboardListener(final Token token) {
+		return new BlackboardWorkflowJobListener(token) {
+
+			@Override
+			protected void onDone(final BlackboardJob job) {
+
+				final STATUS status = getStatusFromEnv(token.getEnv());
+				log.debug("complete phase: " + status);
+				switch (status) {
+				case DATALOAD:
+					token.getEnv().setAttribute(StatusParam, STATUS.DEPTH_N.toString());
+					token.getEnv().setAttribute(DepthParam, "0");
+					token.release("depth_n");
+					break;
+				case DEPTH_N:
+
+					if (log.isDebugEnabled()) {
+						log.debug(String.format("return job parameters: %s=%s, %s=%s", DepthParam, job.getParameters().get(DepthParam), UpdateCounterParam,
+								job.getParameters().get(UpdateCounterParam)));
+					}
+
+					final String counter = job.getParameters().get(UpdateCounterParam);
+					if (StringUtils.isBlank(counter)) {
+						token.getEnv().removeAttribute(StatusParam);
+						token.getEnv().removeAttribute(DepthParam);
+						log.info(String.format("done iteration %s:%s", UpdateCounterParam, 0));
+
+						token.release(Arc.DEFAULT_ARC);
+					} else {
+						log.info(String.format("continue with next iteration %s:%s", UpdateCounterParam, counter));
+						token.release("depth_n");
+					}
+
+					break;
+				}
+			}
+		};
+	}
+
+	public boolean isDebug() {
+		return debug;
+	}
+
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+	}
+
+	public String getOutPathParam() {
+		return outPathParam;
+	}
+
+	public void setOutPathParam(String outPathParam) {
+		this.outPathParam = outPathParam;
+	}
+
+	public String getWorkDir() {
+		return workDir;
+	}
+
+	public void setWorkDir(final String workDir) {
+		this.workDir = workDir;
+	}
+
+	enum STATUS {DATALOAD, DEPTH_N}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/PrepareHostedbyUpdateSQL.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/PrepareHostedbyUpdateSQL.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/PrepareHostedbyUpdateSQL.java	(revision 43444)
@@ -0,0 +1,52 @@
+package eu.dnetlib.msro.workflows.nodes.hostedby;
+
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+import org.antlr.stringtemplate.StringTemplate;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+
+/**
+ * Created by alessia on 01/02/16.
+ */
+public class PrepareHostedbyUpdateSQL extends SimpleJobNode {
+
+	private static final Log log = LogFactory.getLog(PrepareHostedbyUpdateSQL.class);
+	private String updateSqlTemplatePath;
+	private String theQueryParamName = "hostedByUpdateSQL";
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+		final String datasourceId = env.getAttribute(WorkflowsConstants.LOG_DATASOURCE_ID, String.class);
+		final String datasourceNamespacePrefix = env.getAttribute(WorkflowsConstants.DATASOURCE_PREFIX, String.class);
+		final Resource updateSqlTemplate = new ClassPathResource(this.updateSqlTemplatePath);
+		final StringTemplate st = new StringTemplate(IOUtils.toString(updateSqlTemplate.getInputStream()));
+		st.setAttribute("datasourceId", datasourceId);
+		st.setAttribute("datasourceNamespacePrefix", datasourceNamespacePrefix);
+		final String theQuery = st.toString();
+		log.debug("Generated query: " + theQuery);
+		env.setAttribute(this.theQueryParamName, theQuery);
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getUpdateSqlTemplatePath() {
+		return this.updateSqlTemplatePath;
+	}
+
+	public void setUpdateSqlTemplatePath(final String updateSqlTemplatePath) {
+		this.updateSqlTemplatePath = updateSqlTemplatePath;
+	}
+
+	public String getTheQueryParamName() {
+		return this.theQueryParamName;
+	}
+
+	public void setTheQueryParamName(final String theQueryParamName) {
+		this.theQueryParamName = theQueryParamName;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/FinalizeDedupIndexJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/FinalizeDedupIndexJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/FinalizeDedupIndexJobNode.java	(revision 43444)
@@ -0,0 +1,77 @@
+package eu.dnetlib.msro.workflows.nodes.dedup;
+
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.rmi.enabling.ISLookUpDocumentNotFoundException;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import eu.dnetlib.rmi.manager.MSROException;
+import eu.dnetlib.rmi.provision.IndexService;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import static java.lang.String.format;
+
+public class FinalizeDedupIndexJobNode extends BlackboardJobNode {
+
+	private static final Log log = LogFactory.getLog(FinalizeDedupIndexJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	private String actionset;
+
+	@Override
+	protected String obtainServiceId(final Env env) {
+		return getServiceLocator().getServiceId(IndexService.class);
+	}
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		final String indexDsId = getEnvParam(token, "index_id");
+
+		log.info("preparing blackboard job DELETE_BY_QUERY index: " + indexDsId);
+
+		final String backendId = getBackendId(indexDsId);
+		if (StringUtils.isBlank(backendId))
+			throw new MSROException("empty index backend Id");
+
+		job.setAction("DELETE_BY_QUERY");
+		job.getParameters().put("id", indexDsId);
+		job.getParameters().put("backend_Id", backendId);
+		job.getParameters().put("query",
+				buildQuery(getEnvParam(token, "entityType"), getEnvParam(token, "index.feed.timestamp"), getActionset()));
+	}
+
+	private String buildQuery(final String entityType, final String version, final String actionset) {
+		final String query =
+				String.format("__dsversion:{* TO %s} AND oaftype:%s AND actionset:%s", DateUtils.getParsedDateField(version), entityType, actionset);
+
+		log.info("delete by query: " + query);
+
+		return query;
+	}
+
+	private String getEnvParam(final Token token, final String name) throws MSROException {
+		final String value = token.getEnv().getAttribute(name, String.class);
+
+		if (StringUtils.isBlank(value))
+			throw new MSROException(format("unable to finalize index feeding, cannot find property '%s' in the workflow env.", name));
+
+		return value;
+	}
+
+	public String getBackendId(final String indexDsId) throws ISLookUpDocumentNotFoundException, ISLookUpException {
+		return getServiceLocator().getService(ISLookUpService.class).getResourceProfileByQuery(
+				"//RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value='" + indexDsId + "']//BACKEND/text()");
+	}
+
+	public String getActionset() {
+		return actionset;
+	}
+
+	public void setActionset(final String actionset) {
+		this.actionset = actionset;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/FindHostedByJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/FindHostedByJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/FindHostedByJobNode.java	(revision 43444)
@@ -0,0 +1,122 @@
+package eu.dnetlib.msro.workflows.nodes.hostedby;
+
+import java.io.StringReader;
+import java.util.function.UnaryOperator;
+
+import eu.dnetlib.enabling.resultset.factory.ResultSetFactory;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.common.ResultSet;
+import org.dom4j.Document;
+import org.dom4j.Element;
+import org.dom4j.io.SAXReader;
+import org.springframework.beans.factory.annotation.Autowired;
+
+// TODO: Auto-generated Javadoc
+
+/**
+ * The Class FindHostedByJonbNode.
+ */
+public class FindHostedByJobNode extends SimpleJobNode {
+
+	private final String unknown_repo_id = "openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18";
+	/**
+	 * The input epr param.
+	 */
+	private String inputEprParam;
+	/**
+	 * The output epr param.
+	 */
+	private String outputEprParam;
+	/**
+	 * The counters param.
+	 */
+	private String countersParam;
+	/**
+	 * The mapped result set factory.
+	 */
+	@Autowired
+	private ResultSetFactory resultSetFactory;
+
+	/*
+	 * (non-Javadoc)
+	 *
+	 * @see eu.dnetlib.msro.workflows.nodes.SimpleJobNode#execute(com.googlecode.sarasvati.NodeToken)
+	 */
+	@Override
+	protected String execute(final Env env) throws Exception {
+		final ResultSet<?> inputEpr = env.getAttribute(this.inputEprParam, ResultSet.class);
+		final HostedByCounters counters = new HostedByCounters();
+
+		final SAXReader reader = new SAXReader();
+
+		final UnaryOperator<String> hostedByMapFunction = input -> {
+			try {
+				final Document doc = reader.read(new StringReader(input));
+				final Element node = (Element) doc.selectSingleNode("//*[local-name()='hostedBy']");
+				if (node != null) {
+					final String hostedById = node.attributeValue("id");
+					if (!hostedById.equals(FindHostedByJobNode.this.unknown_repo_id)) {
+						counters.increaseCounter(hostedById);
+					}
+				}
+			} catch (final Exception e) {
+
+			}
+
+			return input;
+		};
+
+		final ResultSet<String> epr = this.resultSetFactory.map(inputEpr, String.class,
+				hostedByMapFunction);
+
+		env.setAttribute(this.outputEprParam, epr);
+		env.setAttribute(getCountersParam(), counters);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	/**
+	 * @return the inputEprParam
+	 */
+	public String getInputEprParam() {
+		return this.inputEprParam;
+	}
+
+	/**
+	 * @param inputEprParam the inputEprParam to set
+	 */
+	public void setInputEprParam(final String inputEprParam) {
+		this.inputEprParam = inputEprParam;
+	}
+
+	/**
+	 * @return the outputEprParam
+	 */
+	public String getOutputEprParam() {
+		return this.outputEprParam;
+	}
+
+	/**
+	 * @param outputEprParam the outputEprParam to set
+	 */
+	public void setOutputEprParam(final String outputEprParam) {
+		this.outputEprParam = outputEprParam;
+	}
+
+	/**
+	 * @return the countersParam
+	 */
+	public String getCountersParam() {
+		return this.countersParam;
+	}
+
+	/**
+	 * @param countersParam the countersParam to set
+	 */
+	public void setCountersParam(final String countersParam) {
+		this.countersParam = countersParam;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/HostedByEntry.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/HostedByEntry.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/HostedByEntry.java	(revision 43444)
@@ -0,0 +1,34 @@
+package eu.dnetlib.msro.workflows.nodes.hostedby;
+
+public class HostedByEntry {
+
+	private String id;
+	private String name;
+
+	public HostedByEntry(String id, String name) {
+		this.id = id;
+		this.name = name;
+	}
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(String name) {
+		this.name = name;
+	}
+
+	@Override
+	public String toString() {
+		return "[ id: " + id + ", name: " + name + " ]";
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/QueryUserActionDbJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/QueryUserActionDbJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/QueryUserActionDbJobNode.java	(revision 43444)
@@ -0,0 +1,126 @@
+package eu.dnetlib.msro.workflows.nodes.dedup;
+
+import java.io.IOException;
+import javax.annotation.Resource;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.common.ResultSet;
+import eu.dnetlib.rmi.data.DatabaseService;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class QueryUserActionDbJobNode extends SimpleJobNode {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(QueryUserActionDbJobNode.class);
+
+	private String db;
+
+	private String sql;
+	private String sqlForSize;
+	private String xslt;
+	private String outputEprParam;
+
+	private String dedupConfigSequence;
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		if (StringUtils.isBlank(getDedupConfigSequence())) throw new IllegalArgumentException("empty action set id");
+
+		final String sqlText = String.format(readFromClasspath(getSql()), getDedupConfigSequence());
+		log.debug("executing query: " + sqlText);
+
+		final String sqlTextForsize = String.format(StringUtils.isBlank(getSqlForSize()) ? "" : readFromClasspath(getSqlForSize()), getDedupConfigSequence());
+		if (!sqlTextForsize.isEmpty()) {
+			log.debug("using sql for size: " + sqlTextForsize);
+		}
+
+		ResultSet<String> rs = null;
+
+		final DatabaseService dbService = serviceLocator.getService(DatabaseService.class);
+
+		if (StringUtils.isNotBlank(getXslt())) {
+			final String xsltText = IOUtils.toString(getClass().getResourceAsStream(getXslt()));
+
+			if (StringUtils.isBlank(getSqlForSize())) {
+				rs = dbService.xsltSearchSQL(getDb(), sqlText, xsltText);
+			} else {
+				rs = dbService.alternativeXsltSearchSQL(getDb(), sqlText, sqlTextForsize, xsltText);
+			}
+		} else {
+			if (StringUtils.isBlank(getSqlForSize())) {
+				rs = dbService.searchSQL(getDb(), sqlText);
+			} else {
+				rs = dbService.alternativeSearchSQL(getDb(), sqlText, sqlTextForsize);
+			}
+		}
+
+		log.info(String.format("generated resultset '%s', baseURL: %s", rs.getId(), rs.getBaseUrl()));
+		env.setAttribute(getOutputEprParam(), rs);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	private String readFromClasspath(final String path) throws IOException {
+		return IOUtils.toString(getClass().getResourceAsStream(path));
+	}
+
+	public String getDb() {
+		return db;
+	}
+
+	public void setDb(final String db) {
+		this.db = db;
+	}
+
+	public String getSql() {
+		return sql;
+	}
+
+	public void setSql(final String sql) {
+		this.sql = sql;
+	}
+
+	public String getXslt() {
+		return xslt;
+	}
+
+	public void setXslt(final String xslt) {
+		this.xslt = xslt;
+	}
+
+	public String getOutputEprParam() {
+		return outputEprParam;
+	}
+
+	public void setOutputEprParam(final String outputEprParam) {
+		this.outputEprParam = outputEprParam;
+	}
+
+	public String getSqlForSize() {
+		return sqlForSize;
+	}
+
+	public void setSqlForSize(final String sqlForSize) {
+		this.sqlForSize = sqlForSize;
+	}
+
+	public String getDedupConfigSequence() {
+		return dedupConfigSequence;
+	}
+
+	public void setDedupConfigSequence(final String dedupConfigSequence) {
+		this.dedupConfigSequence = dedupConfigSequence;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/DedupSimilarityToActionsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/DedupSimilarityToActionsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/DedupSimilarityToActionsJobNode.java	(revision 43444)
@@ -0,0 +1,56 @@
+package eu.dnetlib.msro.workflows.nodes.dedup;
+
+import eu.dnetlib.data.proto.DedupSimilarityProtos.DedupSimilarity;
+import eu.dnetlib.data.proto.RelTypeProtos.RelType;
+import eu.dnetlib.data.proto.RelTypeProtos.SubRelType;
+import eu.dnetlib.data.proto.TypeProtos.Type;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class DedupSimilarityToActionsJobNode extends DedupConfigurationLoaderJobNode {
+
+	private static final Log log = LogFactory.getLog(DedupSimilarityToActionsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	private String entityType;
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		super.prepareJob(job, token);
+
+		final String entityType = getEntityType();
+		if (StringUtils.isBlank(entityType))
+			throw new MSROException("unable to find wf param: entityType");
+
+		String cf = "_" + SubRelType.dedupSimilarity + "_" + DedupSimilarity.RelName.isSimilarTo;
+		switch (Type.valueOf(entityType)) {
+
+		case organization:
+			cf = RelType.organizationOrganization + cf;
+			break;
+		case person:
+			cf = RelType.personPerson + cf;
+			break;
+		case result:
+			cf = RelType.resultResult + cf;
+			break;
+		default:
+			throw new MSROException("invalid parameter entityType: " + entityType);
+		}
+
+		log.info("using similarity CF: " + cf);
+		job.getParameters().put("similarityCF", cf);
+		token.getEnv().setAttribute("similarityCF", cf);
+	}
+
+	public String getEntityType() {
+		return entityType;
+	}
+
+	public void setEntityType(final String entityType) {
+		this.entityType = entityType;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/HostedByCounters.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/HostedByCounters.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hostedby/HostedByCounters.java	(revision 43444)
@@ -0,0 +1,36 @@
+package eu.dnetlib.msro.workflows.nodes.hostedby;
+
+import java.io.StringWriter;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import com.google.common.collect.Maps;
+
+public class HostedByCounters {
+
+	private Map<String, Integer> counters = Maps.newHashMap();
+
+	public void increaseCounter(final String dsId) {
+		int val = counters.containsKey(dsId) ? counters.get(dsId) + 1 : 1;
+		counters.put(dsId, val);
+	}
+
+	public Map<String, Integer> getCounters() {
+		return counters;
+	}
+
+	@Override
+	public String toString() {
+		final StringWriter sw = new StringWriter();
+		sw.append("\n");
+		sw.append("**************************************************\n");
+		for (Entry<String, Integer> e : counters.entrySet()) {
+			sw.append(e.getKey());
+			sw.append(" : ");
+			sw.append(Integer.toString(e.getValue()));
+			sw.append("\n");
+		}
+		sw.append("**************************************************\n");
+		return sw.toString();
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dfm/TriggerPrepublicMonitoringScenarioJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dfm/TriggerPrepublicMonitoringScenarioJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dfm/TriggerPrepublicMonitoringScenarioJobNode.java	(revision 43444)
@@ -0,0 +1,66 @@
+package eu.dnetlib.msro.workflows.nodes.dfm;
+
+import java.io.IOException;
+import javax.annotation.Resource;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.monitoring.core.sensors.SensorCreationException;
+import eu.dnetlib.monitoring.core.sensors.SensorFactory;
+import eu.dnetlib.monitoring.core.sensors.collection.SensorWrapper;
+import eu.dnetlib.monitoring.core.sensors.collection.index.solr.SolrSensor;
+import eu.dnetlib.monitoring.core.sensors.collection.kv.redis.RedisSensor;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class TriggerPrepublicMonitoringScenarioJobNode extends SimpleJobNode {
+
+	private final static String SOLR_COLLECTION_POSTFIX = "-index-openaire";
+	private final Log log = LogFactory.getLog(TriggerPrepublicMonitoringScenarioJobNode.class);
+	private String mdFormat;
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	private SolrSensor solrSensor;
+	private RedisSensor redisSensor;
+	private SensorWrapper wrapper;
+
+	@Override
+	protected String execute(final Env env) throws IOException, InstantiationException, IllegalAccessException, InterruptedException,
+			ISLookUpException, SensorCreationException {
+		this.log.info("Execution of TriggerPrepublicMonitoringScenarioJobNode");
+
+		/* Configuring SOLR sensor */
+		final String actualCollection = this.mdFormat + SOLR_COLLECTION_POSTFIX;
+		this.log.info("At this round, the Pre-public monitoring scenario will check against " + actualCollection + " SOLR collection");
+		this.solrSensor = (SolrSensor) SensorFactory.createSensorWithConfiguration(SolrSensor.class, "prepublic", "solr");
+		this.solrSensor.getConfiguration().setParam("collection", actualCollection);
+		this.solrSensor.reinitialize();
+
+		/* Configuring Redis sensor */
+		this.redisSensor = (RedisSensor) SensorFactory.createSensorWithConfiguration(RedisSensor.class, "prepublic", "redis");
+
+		/* Wrapping the two sensors together */
+		this.wrapper = (SensorWrapper) SensorFactory.createSensor(SensorWrapper.class);
+		this.wrapper.registerSensor(this.solrSensor);
+		this.wrapper.registerSensor(this.redisSensor);
+
+		this.log.info("Measuring prepublic scenario");
+		this.wrapper.measure();
+		this.log.info("DONE!");
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getMdFormat() {
+		return this.mdFormat;
+	}
+
+	public void setMdFormat(final String mdFormat) {
+		this.mdFormat = mdFormat;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/IndexDSUpdateJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/IndexDSUpdateJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/IndexDSUpdateJobNode.java	(revision 43444)
@@ -0,0 +1,58 @@
+package eu.dnetlib.msro.workflows.nodes;
+
+import javax.annotation.Resource;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.enabling.ISRegistryException;
+import eu.dnetlib.rmi.enabling.ISRegistryService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class IndexDSUpdateJobNode extends SimpleJobNode {
+
+	public final static String N_RECORDS_PARAM = "blackboard:param:index.status code: 0";
+	private static final Log log = LogFactory.getLog(IndexDSUpdateJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	@Override
+	public String execute(final Env env) throws Exception {
+
+		final String dsId = env.getAttribute("index_id", String.class);
+		final String version = env.getAttribute("index.feed.timestamp", String.class);
+
+		try {
+			final int count = Integer.parseInt(env.getAttribute(N_RECORDS_PARAM, String.class));
+			log.info("updating indexDS: " + dsId + " version: " + version + " record count: " + count);
+			updateIndexDS(dsId, version, count);
+		} catch (NumberFormatException e) {
+			log.warn("unable to find index feeding details in the workflow env, skippind DS update");
+		}
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	/**
+	 * method updates the given indexDataStructureId INDEX_SIZE, INDEX_LAST_UPDATE
+	 *
+	 * @param dsId
+	 * @param version
+	 * @param count
+	 * @return true if the update was performed successfully, false otherwise
+	 * @throws ISRegistryException
+	 */
+	private boolean updateIndexDS(final String dsId, final String version, final long count) throws ISRegistryException {
+
+		final String xquery = "for $x in collection('')/RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + dsId + "']"
+				+ " return update value $x//INDEX_SIZE with " + String.valueOf(count) + ","
+				+ "for $x in collection('')/RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + dsId + "']"
+				+ " return update value $x//INDEX_LAST_UPDATE with '" + version + "' ";
+
+		log.debug("\n\n updating indexDataStructure: " + xquery + "\n\n");
+
+		return serviceLocator.getService(ISRegistryService.class).executeXUpdate(xquery);
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dfm/TriggerIndexMonitoringScenarioJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dfm/TriggerIndexMonitoringScenarioJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dfm/TriggerIndexMonitoringScenarioJobNode.java	(revision 43444)
@@ -0,0 +1,51 @@
+package eu.dnetlib.msro.workflows.nodes.dfm;
+
+import java.io.IOException;
+import javax.annotation.Resource;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.monitoring.core.sensors.SensorCreationException;
+import eu.dnetlib.monitoring.core.sensors.SensorFactory;
+import eu.dnetlib.monitoring.core.sensors.collection.index.solr.SolrSensor;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class TriggerIndexMonitoringScenarioJobNode extends SimpleJobNode {
+
+	private static final Log log = LogFactory.getLog(TriggerIndexMonitoringScenarioJobNode.class);
+	private static final String SOLR_COLLECTION_POSTFIX = "-index-openaire";
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+	private String mdFormat;
+	private SolrSensor solrSensor;
+
+	@Override
+	protected String execute(final Env env) throws IOException, InstantiationException, IllegalAccessException, InterruptedException,
+			ISLookUpException, SensorCreationException {
+
+		/* Configuring SOLR sensor */
+		final String actualCollection = this.mdFormat + SOLR_COLLECTION_POSTFIX;
+		log.info("Executing on " + actualCollection + " SOLR collection");
+		this.solrSensor = (SolrSensor) SensorFactory.createSensorWithConfiguration(SolrSensor.class, "index", "solr2");
+		this.solrSensor.getConfiguration().setParam("collection", actualCollection);
+		this.solrSensor.reinitialize();
+		log.info("Measuring...");
+		this.solrSensor.measure();
+		log.info("DONE!");
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getMdFormat() {
+		return this.mdFormat;
+	}
+
+	public void setMdFormat(final String mdFormat) {
+		this.mdFormat = mdFormat;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/OafToIndexRecordFactory.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/OafToIndexRecordFactory.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/OafToIndexRecordFactory.java	(revision 43444)
@@ -0,0 +1,63 @@
+package eu.dnetlib.msro.workflows.nodes.index;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.StringWriter;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.miscutils.functional.xml.ApplyXslt;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import org.apache.commons.io.IOUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Required;
+import org.springframework.core.io.ClassPathResource;
+
+public class OafToIndexRecordFactory {
+
+	private ClassPathResource layoutToRecord;
+
+	@Autowired
+	private UniqueServiceLocator serviceLocator;
+
+	public ApplyXslt newTransformer(final String format) throws IOException, TransformerException, ISLookUpException {
+		final StreamResult layoutToXsltXslt = getTransformer(format);
+
+		return new ApplyXslt(layoutToXsltXslt.getWriter().toString());
+	}
+
+	private StreamResult getTransformer(final String format) throws IOException, TransformerException, ISLookUpException {
+		final TransformerFactory factory = TransformerFactory.newInstance();
+		final Transformer layoutTransformer = factory.newTransformer(readLayoutToRecord());
+
+		final StreamResult layoutToXsltXslt = new StreamResult(new StringWriter());
+
+		layoutTransformer.setParameter("format", format);
+		layoutTransformer.transform(new StreamSource(new StringReader(getLayoutSource(format))), layoutToXsltXslt);
+		return layoutToXsltXslt;
+	}
+
+	private StreamSource readLayoutToRecord() throws IOException {
+		return new StreamSource(new StringReader(IOUtils.toString(layoutToRecord.getInputStream())));
+	}
+
+	private String getLayoutSource(final String format) throws ISLookUpException {
+		return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(
+				"collection('/db/DRIVER/MDFormatDSResources/MDFormatDSResourceType')[.//NAME='" + format + "']//LAYOUT[@name='index']");
+	}
+
+	public ClassPathResource getLayoutToRecord() {
+		return layoutToRecord;
+	}
+
+	@Required
+	public void setLayoutToRecord(final ClassPathResource layoutToRecord) {
+		this.layoutToRecord = layoutToRecord;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/FeedMissingClaimsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/FeedMissingClaimsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/FeedMissingClaimsJobNode.java	(revision 43444)
@@ -0,0 +1,154 @@
+package eu.dnetlib.msro.workflows.nodes.index;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+import javax.annotation.Resource;
+
+import com.google.common.base.Function;
+import eu.dnetlib.data.index.CloudIndexClient;
+import eu.dnetlib.data.index.CloudIndexClientException;
+import eu.dnetlib.data.index.CloudIndexClientFactory;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.AsyncJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.openaire.api.RecentPublicationsQueue;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.solr.common.SolrInputDocument;
+import org.dom4j.io.SAXReader;
+import org.springframework.beans.factory.annotation.Required;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.io.ClassPathResource;
+
+/**
+ * Created by michele on 15/12/15.
+ */
+public class FeedMissingClaimsJobNode extends AsyncJobNode {
+
+	private static final Log log = LogFactory.getLog(FeedMissingClaimsJobNode.class);
+	private RecentPublicationsQueue queue;
+	private OafToIndexRecordFactory oafToIndexRecordFactory;
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	@Value(value = "${openaireplus.msro.api.findSolrIndexUrl.xquery}")
+	private ClassPathResource findSolrIndexUrl;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		final String format = env.getAttribute("format", String.class);
+		final String coll = format + "-index-openaire";
+		final String indexDsId = env.getAttribute("index_id", String.class);
+		final String baseUrl = calculateIndexBaseUrl();
+
+		CloudIndexClient idxClient = null;
+
+		try {
+			final List<SolrInputDocument> toFeed = new ArrayList<>();
+			final List<String> toDeleteFromCache = new ArrayList<>();
+
+			final SAXReader reader = new SAXReader();
+			final Function<String, String> xslt = (Function<String, String>) oafToIndexRecordFactory.newTransformer(format);
+
+			idxClient = CloudIndexClientFactory.newIndexClient(baseUrl, coll, false);
+
+			for (final String record : this.queue) {
+				final String id = reader.read(new StringReader(record)).valueOf("//*[local-name() = 'objIdentifier']");
+				if (idxClient.isRecordIndexed(id)) {
+					toFeed.add(idxClient.prepareSolrDocument(record, indexDsId, xslt));
+				} else {
+					toDeleteFromCache.add(id);
+				}
+			}
+
+			idxClient.feed(toFeed, null);
+			this.queue.remove(toDeleteFromCache);
+
+		} catch (final Throwable e) {
+			log.error("Error feeding missing claims", e);
+			throw new MSROException("Error feeding missing claims: " + e.getMessage(), e);
+		} finally {
+			if (idxClient != null) {
+				idxClient.close();
+			}
+		}
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	public RecentPublicationsQueue getQueue() {
+		return this.queue;
+	}
+
+	@Required
+	public void setQueue(final RecentPublicationsQueue queue) {
+		this.queue = queue;
+	}
+
+	public OafToIndexRecordFactory getOafToIndexRecordFactory() {
+		return this.oafToIndexRecordFactory;
+	}
+
+	@Required
+	public void setOafToIndexRecordFactory(final OafToIndexRecordFactory oafToIndexRecordFactory) {
+		this.oafToIndexRecordFactory = oafToIndexRecordFactory;
+	}
+
+	private String calculateIndexBaseUrl() throws Exception {
+		final String query = IOUtils.toString(this.findSolrIndexUrl.getInputStream());
+		return this.serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(query);
+	}
+
+	private class IsNotIndexed implements com.google.common.base.Predicate<String> {
+
+		private final CloudIndexClient idxClient;
+
+		private final SAXReader reader = new SAXReader();
+
+		public IsNotIndexed(final CloudIndexClient idxClient) {
+			this.idxClient = idxClient;
+		}
+
+		@Override
+		public boolean apply(final String s) {
+			try {
+				final String id = this.reader.read(new StringReader(s)).valueOf("//*[local-name() = 'objIdentifier']");
+				return !this.idxClient.isRecordIndexed(id);
+			} catch (final Throwable e) {
+				log.error("Error searching record: " + s, e);
+				throw new RuntimeException(e);
+			}
+		}
+
+	}
+
+	private class CreateSolrDocument implements Function<String, SolrInputDocument> {
+
+		private final CloudIndexClient idxClient;
+		private final String indexDsId;
+		private final Function<String, String> toIndexRecord;
+
+		public CreateSolrDocument(final CloudIndexClient idxClient, final String indexDsId, final Function<String, String> toIndexRecord) {
+			this.idxClient = idxClient;
+			this.indexDsId = indexDsId;
+			this.toIndexRecord = toIndexRecord;
+		}
+
+		@Override
+		public SolrInputDocument apply(final String s) {
+			try {
+				return this.idxClient.prepareSolrDocument(s, this.indexDsId, this.toIndexRecord);
+			} catch (final CloudIndexClientException e) {
+				throw new RuntimeException(e);
+			}
+		}
+
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/objectstore/DownloadFromMetadataOpenaire.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/objectstore/DownloadFromMetadataOpenaire.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/objectstore/DownloadFromMetadataOpenaire.java	(revision 43444)
@@ -0,0 +1,82 @@
+package eu.dnetlib.msro.workflows.nodes.objectstore;
+
+import java.util.Map;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.blackboard.BlackboardWorkflowJobListener;
+import eu.dnetlib.msro.workflows.nodes.download.DownloadFromMetadataJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+import eu.dnetlib.rmi.data.ObjectStoreService;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class DownloadFromMetadataOpenaire extends DownloadFromMetadataJobNode {
+
+	private static final Log log = LogFactory.getLog(DownloadFromMetadataOpenaire.class);
+
+	private String basepath;
+
+	@Autowired
+	private UniqueServiceLocator serviceLocator;
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		if (this.getBasepath() != null && this.getBasepath().length() > 0) {
+			job.getParameters().put("basePath", this.basepath);
+		}
+
+		final int size = this.serviceLocator.getService(ObjectStoreService.class).getSize(getObjectStoreID());
+		log.debug(String.format("objectStore %s has size %s", getObjectStoreID(), size));
+		token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "oldTotal", size);
+
+		super.prepareJob(job, token);
+	}
+
+	/*
+	 * (non-Javadoc)
+	 *
+	 * @see eu.dnetlib.msro.workflows.nodes.BlackboardJobNode#generateBlackboardListener(com.googlecode.sarasvati.Engine,
+	 * com.googlecode.sarasvati.NodeToken)
+	 */
+	@Override
+	protected BlackboardWorkflowJobListener generateBlackboardListener(final Token token) {
+		return new BlackboardWorkflowJobListener(token) {
+
+			@Override
+			protected void responseToEnv(final Env env, final Map<String, String> responseParams) {
+				env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "total", responseParams.get("total"));
+
+				final String base64 = responseParams.get("report");
+				if (StringUtils.isNotBlank(base64) && Base64.isBase64(base64.getBytes())) {
+					final String report = new String(Base64.decodeBase64(base64.getBytes()));
+					log.info("found download report");
+					log.debug(report);
+					env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "report", report);
+				} else {
+					log.warn("unable to find or decode download report");
+				}
+			}
+		};
+	}
+
+	/**
+	 * @return the basepath
+	 */
+	public String getBasepath() {
+		return this.basepath;
+	}
+
+	/**
+	 * @param basepath the basepath to set
+	 */
+	public void setBasepath(final String basepath) {
+		this.basepath = basepath;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/objectstore/ArxivIteratorRepository.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/objectstore/ArxivIteratorRepository.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/objectstore/ArxivIteratorRepository.java	(revision 43444)
@@ -0,0 +1,51 @@
+package eu.dnetlib.msro.workflows.nodes.objectstore;
+
+import java.io.File;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions;
+import eu.dnetlib.rmi.data.ObjectStoreFile;
+import eu.dnetlib.rmi.data.Protocols;
+
+public class ArxivIteratorRepository extends IteratorRepository {
+
+	private final Pattern pattern;
+
+	public ArxivIteratorRepository(final Iterable<File> input, final String repositoryPrefix, final String oaiPrefix) {
+		super(input, repositoryPrefix, oaiPrefix);
+		this.pattern = Pattern.compile("[0-9]");
+
+	}
+
+	@Override
+	String generateNextElement() {
+		try {
+			final String inputname = this.input.next().getCanonicalPath();
+			final String[] values = inputname.split("/");
+			if (values.length > 1) {
+				final ObjectStoreFile info = new ObjectStoreFile();
+				String fileName = values[values.length - 1].replace(".pdf", "");
+				final Matcher matcher = this.pattern.matcher(fileName);
+				if (matcher.find()) {
+					if (matcher.start() != 0) {
+						fileName = fileName.substring(0, matcher.start()) + "/" + fileName.substring(matcher.start());
+					}
+				}
+				info.setDownloadedURL("http://arxiv.org/abs/" + fileName);
+				info.setAccessProtocol(Protocols.None);
+				// String value = "oai:arXiv.org:" + fileName;
+				final String value = this.oaiPrefix + fileName;
+				final String resultID = this.repositoryPrefix + "::" + DnetXsltFunctions.md5(value);
+				info.setObjectID(resultID + "::" + DnetXsltFunctions.md5(info.getDownloadedURL()));
+				info.setMimeType("pdf");
+				info.setURI(inputname);
+				return info.toJSON();
+			}
+		} catch (final Exception e) {
+			return null;
+		}
+		return null;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/FinalizeIndexJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/FinalizeIndexJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/FinalizeIndexJobNode.java	(revision 43444)
@@ -0,0 +1,66 @@
+package eu.dnetlib.msro.workflows.nodes.index;
+
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.rmi.enabling.ISLookUpDocumentNotFoundException;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import eu.dnetlib.rmi.manager.MSROException;
+import eu.dnetlib.rmi.provision.IndexService;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import static java.lang.String.format;
+
+public class FinalizeIndexJobNode extends BlackboardJobNode {
+
+	private static final Log log = LogFactory.getLog(FinalizeIndexJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	@Override
+	protected String obtainServiceId(final Env env) {
+		return getServiceLocator().getServiceId(IndexService.class);
+	}
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		final String indexDsId = getEnvParam(token.getEnv(), "index_id");
+
+		log.info("preparing blackboard job DELETE_BY_QUERY index: " + indexDsId);
+
+		final String backendId = getBackendId(indexDsId);
+		if (StringUtils.isBlank(backendId)) { throw new MSROException("empty index backend Id"); }
+
+		job.setAction("DELETE_BY_QUERY");
+		job.getParameters().put("id", indexDsId);
+		job.getParameters().put("backend_Id", backendId);
+		job.getParameters().put("query", buildQuery(getEnvParam(token.getEnv(), "index.feed.timestamp")));
+	}
+
+	private String buildQuery(final String version) {
+		final String query =
+				String.format("__dsversion:{* TO %s}", DateUtils.getParsedDateField(version));
+
+		log.info("delete by query: " + query);
+
+		return query;
+	}
+
+	private String getEnvParam(final Env env, final String name) throws MSROException {
+		final String value = env.getAttribute(name, String.class);
+
+		if (StringUtils
+				.isBlank(value)) { throw new MSROException(format("unable to finalize index feeding, cannot find property '%s' in the workflow env.", name)); }
+
+		return value;
+	}
+
+	public String getBackendId(final String indexDsId) throws ISLookUpDocumentNotFoundException, ISLookUpException {
+		return getServiceLocator().getService(ISLookUpService.class).getResourceProfileByQuery(
+				"//RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value='" + indexDsId + "']//BACKEND/text()");
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/objectstore/RepositoryNoURLIterator.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/objectstore/RepositoryNoURLIterator.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/objectstore/RepositoryNoURLIterator.java	(revision 43444)
@@ -0,0 +1,38 @@
+package eu.dnetlib.msro.workflows.nodes.objectstore;
+
+import java.io.File;
+
+import eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions;
+import eu.dnetlib.rmi.data.ObjectStoreFile;
+import eu.dnetlib.rmi.data.Protocols;
+
+public class RepositoryNoURLIterator extends IteratorRepository {
+
+	public RepositoryNoURLIterator(final Iterable<File> input, final String repositoryPrefix, final String oaiPrefix) {
+		super(input, repositoryPrefix, oaiPrefix);
+	}
+
+	@Override
+	String generateNextElement() {
+		try {
+			final String inputname = this.input.next().getCanonicalPath();
+			final String[] values = inputname.split("/");
+			if (values.length > 1) {
+				final ObjectStoreFile info = new ObjectStoreFile();
+				final String fileName = values[values.length - 1].replace(".xml", "");
+				info.setAccessProtocol(Protocols.None);
+				// String value = "oai:europepmc.org:" + fileName;
+				final String value = this.oaiPrefix + fileName;
+				final String resultID = this.repositoryPrefix + "::" + DnetXsltFunctions.md5(value);
+				info.setObjectID(resultID);
+				info.setMimeType("pdf");
+				info.setURI(inputname);
+				return info.toJSON();
+			}
+		} catch (final Exception e) {
+			return null;
+		}
+		return null;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/objectstore/IteratorRepository.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/objectstore/IteratorRepository.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/objectstore/IteratorRepository.java	(revision 43444)
@@ -0,0 +1,42 @@
+package eu.dnetlib.msro.workflows.nodes.objectstore;
+
+import java.io.File;
+import java.util.Iterator;
+
+public abstract class IteratorRepository implements Iterator<String>, Iterable<String> {
+
+	protected Iterator<File> input;
+	protected Iterable<File> inputIterable;
+	protected String repositoryPrefix;
+	protected String oaiPrefix;
+
+	public IteratorRepository(final Iterable<File> input, final String repositoryPrefix, final String oaiPrefix) {
+		this.inputIterable = input;
+		this.repositoryPrefix = repositoryPrefix;
+		this.oaiPrefix = oaiPrefix;
+	}
+
+	abstract String generateNextElement();
+
+	@Override
+	public boolean hasNext() {
+		return input.hasNext();
+	}
+
+	@Override
+	public String next() {
+		return generateNextElement();
+	}
+
+	@Override
+	public void remove() {
+		// TODO Auto-generated method stub
+	}
+
+	@Override
+	public Iterator<String> iterator() {
+		this.input = inputIterable.iterator();
+		return this;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/ContextLoader.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/ContextLoader.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/ContextLoader.java	(revision 43444)
@@ -0,0 +1,48 @@
+package eu.dnetlib.msro.workflows.nodes.index;
+
+import javax.annotation.Resource;
+
+import com.google.common.base.Joiner;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.miscutils.functional.xml.XMLIndenter;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+
+public class ContextLoader {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(ContextLoader.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	private String xquery;
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	public String load() throws ISLookUpException {
+
+		log.info("loading ContextDSResources: " + getXquery());
+
+		final StringBuilder sb = new StringBuilder("<ContextDSResources>");
+		Joiner.on("").appendTo(sb, this.serviceLocator.getService(ISLookUpService.class).quickSearchProfile(getXquery()));
+		sb.append("</ContextDSResources>");
+
+		log.debug("got ContextDSResources: \n" + XMLIndenter.indent(sb.toString()));
+
+		return sb.toString();
+	}
+
+	public String getXquery() {
+		return this.xquery;
+	}
+
+	@Required
+	public void setXquery(final String xquery) {
+		this.xquery = xquery;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/contexts/ContextUtils.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/contexts/ContextUtils.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/contexts/ContextUtils.java	(revision 43444)
@@ -0,0 +1,157 @@
+package eu.dnetlib.msro.workflows.nodes.contexts;
+
+import java.io.StringReader;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Maps;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.*;
+import org.dom4j.io.SAXReader;
+
+/**
+ * Created by claudio on 01/03/16.
+ */
+public class ContextUtils {
+
+	private static final Log log = LogFactory.getLog(ContextUtils.class);
+
+	public static ContextDesc getContext(final Iterable<String> it, final String contextId, final String contextLabel, final String contextType) throws
+			DocumentException {
+
+		return new ContextUtils().getContextDesc(it, contextId, contextLabel, contextType);
+	}
+
+	public static Function<Entry<String, String>, String> getContextRowTransformer() {
+
+		return new Function<Entry<String, String>, String>() {
+
+			private final SAXReader reader = new SAXReader();
+
+			@Override
+			public String apply(final Entry<String, String> e) {
+				try {
+					final Document docFundingPath = reader.read(new StringReader(e.getValue()));
+
+					final Element root = DocumentHelper.createElement("ROWS");
+
+					final Map<String, String> fundingFields = Maps.newLinkedHashMap();
+					fundingFields.put("_dnet_resource_identifier_", e.getKey());
+					fundingFields.put("id", e.getKey());
+					fundingFields.put("path", e.getValue());
+					fundingFields.put("jurisdiction", docFundingPath.valueOf("//funder/jurisdiction"));
+					final String desc = docFundingPath.valueOf("(//*[starts-with(local-name(),'funding_level_')])[1]/description");
+					if (StringUtils.isNotBlank(desc)) {
+						fundingFields.put("description", desc);
+					}
+					final Map<String, String> orgFields = findFunderInfo(docFundingPath);
+					final String orgId = orgFields.get("id");
+					if (StringUtils.isNotBlank(orgId)) {
+						addRow(root, "organizations", orgFields);
+						fundingFields.put("funder", orgId);
+					}
+
+					addRow(root, "fundingpaths", fundingFields);
+
+					if (log.isDebugEnabled()) {
+						log.debug("Db entries: " + root.asXML());
+					}
+
+					return root.asXML();
+				} catch (DocumentException e1) {
+					log.error("Error parsing xml", e1);
+					throw new RuntimeException("Error parsing xml", e1);
+				}
+			}
+
+			private Map<String, String> findFunderInfo(final Document doc) {
+				final Map<String, String> res = Maps.newLinkedHashMap();
+				res.put("_dnet_resource_identifier_", doc.valueOf("//funder/id"));
+				res.put("id", doc.valueOf("//funder/id"));
+				res.put("legalshortname", doc.valueOf("//funder/shortname"));
+				res.put("legalname", doc.valueOf("//funder/name"));
+				return res;
+			}
+
+			private void addRow(final Element root, final String table, final Map<String, String> fields) {
+				final Element row = root.addElement("ROW");
+				row.addAttribute("table", table);
+				for (Entry<String, String> e : fields.entrySet()) {
+					final Element pathField = row.addElement("FIELD");
+					pathField.addAttribute("name", e.getKey());
+					pathField.setText(e.getValue());
+				}
+			}
+		};
+	}
+
+	private ContextDesc getContextDesc(final Iterable<String> it, final String contextId, final String contextLabel, final String contextType) throws
+			DocumentException {
+		final ContextDesc context = new ContextDesc(contextId, contextLabel, contextType);
+		final SAXReader reader = new SAXReader();
+
+		for (String s : it) {
+			populateContext(context, reader, s);
+		}
+		return context;
+	}
+
+	private void populateContext(final ContextDesc context, final SAXReader reader, final String s) throws DocumentException {
+		final Document doc = reader.read(new StringReader(s));
+
+		for (Object o : doc.selectNodes("//fundingtree")) {
+			final Element treeNode = (Element) o;
+
+			final String funder = treeNode.valueOf("./funder/id");
+			if (!context.getDbEntries().containsKey(funder)) {
+				log.info("Found funder: " + funder);
+				context.getDbEntries().put(funder, treeNode.selectSingleNode("./funder").asXML());
+				log.debug("db entry: " + context.getDbEntries().get(funder));
+			}
+
+			final String openaireId = treeNode.valueOf("./*[starts-with(local-name(),'funding_level_')]/id");
+			if (!context.getDbEntries().containsKey(openaireId) && StringUtils.isNotBlank(openaireId)) {
+				log.info("Found funding: " + openaireId);
+				context.getDbEntries().put(openaireId, treeNode.asXML());
+				log.debug("db entry: " + context.getDbEntries().get(openaireId));
+				final Node node0 = treeNode.selectSingleNode(".//funding_level_0");
+				if (node0 != null) {
+					final ContextPart part = calculatePart(node0);
+					if (context.getCategories().containsKey(part.getId())) {
+						for (ContextPart p : part.getParts().values()) {
+							context.getCategories().get(part.getId()).addPart(p);
+							;
+						}
+					} else {
+						context.getCategories().put(part.getId(), part);
+					}
+				}
+			}
+		}
+	}
+
+	private ContextPart calculatePart(final Node node) {
+
+		// final String newId = contextId + "::" + StringUtils.substringAfter(node.valueOf("./id"), "::");
+		// ids are built as: nsPrefix :: funderID :: fundingLevel0ID :: etc etc, hence it seems we might not need the contextId parameter.
+		final String newId = StringUtils.substringAfter(node.valueOf("./id"), "::");
+
+		final ContextPart part = new ContextPart(newId, node.valueOf("./description"));
+
+		part.getParams().put("name", node.valueOf("./name"));
+		part.getParams().put("openaireId", node.valueOf("./id"));
+		part.getParams().put("class", node.valueOf("./class"));
+
+		final Element parent = node.getParent() != null ? node.getParent().getParent() : null;
+
+		if (parent != null && parent.getName().startsWith("funding_level_")) {
+			final ContextPart p = calculatePart(parent);
+			part.getParts().put(p.getId(), p);
+		}
+		return part;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/contexts/SaveContextProfileJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/contexts/SaveContextProfileJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/contexts/SaveContextProfileJobNode.java	(revision 43444)
@@ -0,0 +1,80 @@
+package eu.dnetlib.msro.workflows.nodes.contexts;
+
+import java.io.StringReader;
+import java.util.List;
+import javax.annotation.Resource;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import eu.dnetlib.rmi.enabling.ISRegistryException;
+import eu.dnetlib.rmi.enabling.ISRegistryService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.DocumentException;
+import org.dom4j.Element;
+import org.dom4j.Node;
+import org.dom4j.io.SAXReader;
+
+public class SaveContextProfileJobNode extends SimpleJobNode {
+
+	private static final Log log = LogFactory.getLog(SaveContextProfileJobNode.class);
+	private String contextObj;
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+		final ContextDesc desc = env.getAttribute(this.contextObj, ContextDesc.class);
+
+		final String xquery = "for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') " + "where $x//CONFIGURATION/context[@id='"
+				+ desc.getId() + "' " + "and @type='" + desc.getType() + "'] " + "return $x";
+		final List<String> list = this.serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery);
+
+		if (list.isEmpty()) {
+			registerNewProfile(desc);
+		} else {
+			updateProfile(list.get(0), desc);
+		}
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	private boolean updateProfile(final String profile, final ContextDesc desc) throws DocumentException, ISRegistryException {
+		final SAXReader reader = new SAXReader();
+		final Document doc = reader.read(new StringReader(profile));
+		final String profId = doc.valueOf("//HEADER/RESOURCE_IDENTIFIER/@value");
+		final Element ctxElem = (Element) doc.selectSingleNode("//CONFIGURATION/context[@id='" + desc.getId() + "' and @type='" + desc.getType() + "']");
+
+		for (final ContextPart cat : desc.getCategories().values()) {
+			final Node catElem = ctxElem.selectSingleNode("./category[@id=\"" + cat.getId() + "\"]");
+			if (catElem != null) {
+				catElem.detach();
+			}
+			ctxElem.add(cat.asDomElement("category"));
+		}
+		log.info("registering profile context " + desc.getId());
+		return this.serviceLocator.getService(ISRegistryService.class).updateProfile(profId, doc.asXML(), "ContextDSResourceType");
+	}
+
+	private String registerNewProfile(final ContextDesc desc) throws DocumentException, ISRegistryException {
+		final SAXReader reader = new SAXReader();
+		final Document doc = reader.read(getClass().getResourceAsStream(
+				"/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xml/contextProfile.xml"));
+		((Element) doc.selectSingleNode("//CONFIGURATION")).add(desc.asDomElement());
+		log.info("updating profile context " + desc.getId());
+		return this.serviceLocator.getService(ISRegistryService.class).registerProfile(doc.asXML());
+	}
+
+	public String getContextObj() {
+		return this.contextObj;
+	}
+
+	public void setContextObj(final String contextObj) {
+		this.contextObj = contextObj;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/objectstore/PUMAIteratorRepository.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/objectstore/PUMAIteratorRepository.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/objectstore/PUMAIteratorRepository.java	(revision 43444)
@@ -0,0 +1,36 @@
+package eu.dnetlib.msro.workflows.nodes.objectstore;
+
+import java.io.File;
+
+import eu.dnetlib.rmi.data.ObjectStoreFile;
+import eu.dnetlib.rmi.data.Protocols;
+
+public class PUMAIteratorRepository extends IteratorRepository {
+
+	public PUMAIteratorRepository(final Iterable<File> input, final String repositoryPrefix, final String oaiPrefix) {
+		super(input, repositoryPrefix, oaiPrefix);
+		// TODO Auto-generated constructor stub
+	}
+
+	@Override
+	String generateNextElement() {
+		try {
+			final String inputname = this.input.next().getCanonicalPath();
+			final String[] values = inputname.split("/");
+			if (values.length > 1) {
+				final ObjectStoreFile info = new ObjectStoreFile();
+				final String fileName = values[values.length - 1].replace(".pdf", "");
+				info.setAccessProtocol(Protocols.None);
+				info.setObjectID(fileName);
+				info.setDownloadedURL("  ");
+				info.setMimeType("pdf");
+				info.setURI(inputname);
+				return info.toJSON();
+			}
+		} catch (final Exception e) {
+			return null;
+		}
+		return null;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/objectstore/ImportFilesJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/objectstore/ImportFilesJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/objectstore/ImportFilesJobNode.java	(revision 43444)
@@ -0,0 +1,103 @@
+package eu.dnetlib.msro.workflows.nodes.objectstore;
+
+import java.io.File;
+import java.util.List;
+
+import eu.dnetlib.enabling.resultset.client.ResultSetClient;
+import eu.dnetlib.enabling.resultset.factory.ResultSetFactory;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider;
+import eu.dnetlib.rmi.common.ResultSet;
+import eu.dnetlib.rmi.data.ObjectStoreService;
+import eu.dnetlib.rmi.data.Protocols;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class ImportFilesJobNode extends BlackboardJobNode {
+
+	private String basePath;
+	private String repositoryPrefix;
+	private String oaiPrefix;
+	private String objectStoreId;
+	private String contentDescription;
+
+	@Autowired
+	private ResultSetFactory resultSetFactory;
+
+	@Autowired
+	private ResultSetClient resultSetClient;
+
+	public String getRepositoryPrefix() {
+		return this.repositoryPrefix;
+	}
+
+	public void setRepositoryPrefix(final String repositoryPrefix) {
+		this.repositoryPrefix = repositoryPrefix;
+	}
+
+	public String getBasePath() {
+		return this.basePath;
+	}
+
+	public void setBasePath(final String basePath) {
+		this.basePath = basePath;
+	}
+
+	@Override
+	protected String obtainServiceId(final Env env) {
+		return getServiceLocator().getServiceId(ObjectStoreService.class);
+	}
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		final File dir = new File(this.basePath);
+
+		System.out.println("Getting all files in " + dir.getCanonicalPath() + " including those in subdirectories");
+		final List<File> files = (List<File>) FileUtils.listFiles(dir, TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE);
+		IteratorRepository irepo;
+		if (this.repositoryPrefix.equals("od________18")) {
+			irepo = new ArxivIteratorRepository(files, this.repositoryPrefix, this.oaiPrefix);
+		} else if (this.repositoryPrefix.equals("od______2367")) {
+			irepo = new PUMAIteratorRepository(files, this.repositoryPrefix, this.oaiPrefix);
+		} else {
+			irepo = new RepositoryNoURLIterator(files, this.repositoryPrefix, this.oaiPrefix);
+		}
+		final ResultSet<String> epr = this.resultSetFactory.createResultSet(irepo);
+
+		token.setProgressProvider(new ResultsetProgressProvider(epr, this.resultSetClient));
+
+		job.setAction("FEED");
+		job.getParameters().put("obsID", getObjectStoreId());
+		job.getParameters().put("mime", getContentDescription());
+		job.getParameters().put("protocol", Protocols.File_System.toString());
+		job.getParameters().put("epr", epr.toJson());
+	}
+
+	public String getObjectStoreId() {
+		return this.objectStoreId;
+	}
+
+	public void setObjectStoreId(final String objectStoreId) {
+		this.objectStoreId = objectStoreId;
+	}
+
+	public String getContentDescription() {
+		return this.contentDescription;
+	}
+
+	public void setContentDescription(final String contentDescription) {
+		this.contentDescription = contentDescription;
+	}
+
+	public String getOaiPrefix() {
+		return this.oaiPrefix;
+	}
+
+	public void setOaiPrefix(final String oaiPrefix) {
+		this.oaiPrefix = oaiPrefix;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/FindVocabularyEntriesJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/FindVocabularyEntriesJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/FindVocabularyEntriesJobNode.java	(revision 43444)
@@ -0,0 +1,48 @@
+package eu.dnetlib.msro.workflows.nodes;
+
+import java.util.List;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import org.apache.commons.io.IOUtils;
+import org.springframework.beans.factory.annotation.Required;
+import org.springframework.core.io.Resource;
+
+public class FindVocabularyEntriesJobNode extends SimpleJobNode {
+
+	private String outputEprParam = "epr";
+
+	@javax.annotation.Resource
+	private UniqueServiceLocator serviceLocator;
+
+	private Resource xquery;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+		final String query = IOUtils.toString(this.xquery.getInputStream());
+		final List<String> epr = this.serviceLocator.getService(ISLookUpService.class).quickSearchProfile(query);
+		env.setAttribute(getOutputEprParam(), epr);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getOutputEprParam() {
+		return this.outputEprParam;
+	}
+
+	public void setOutputEprParam(final String outputEprParam) {
+		this.outputEprParam = outputEprParam;
+	}
+
+	public Resource getXquery() {
+		return this.xquery;
+	}
+
+	@Required
+	public void setXquery(final Resource xquery) {
+		this.xquery = xquery;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/FindExistingRepositoriesJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/FindExistingRepositoriesJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/FindExistingRepositoriesJobNode.java	(revision 43444)
@@ -0,0 +1,50 @@
+package eu.dnetlib.msro.workflows.nodes;
+
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Resource;
+
+import com.google.common.collect.Maps;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+
+public class FindExistingRepositoriesJobNode extends SimpleJobNode {
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	private String existingReposParam = "existingRepos";
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+		final String xquery =
+				"for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType') " +
+						"return concat (" +
+						"$x//RESOURCE_IDENTIFIER/@value, " +
+						"'@@@', " +
+						"$x//EXTRA_FIELDS/FIELD[./key='OpenAireDataSourceId']/value)";
+		final List<String> list = this.serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery);
+
+		final Map<String, String> map = Maps.newHashMap();
+		for (final String s : list) {
+			final String[] arr = s.split("@@@");
+			if (arr.length == 2) {
+				map.put(arr[1], arr[0]);
+			}
+		}
+		env.setAttribute(getExistingReposParam(), map);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getExistingReposParam() {
+		return this.existingReposParam;
+	}
+
+	public void setExistingReposParam(final String existingReposParam) {
+		this.existingReposParam = existingReposParam;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/UpdateRepositoryProfilesJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/UpdateRepositoryProfilesJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/UpdateRepositoryProfilesJobNode.java	(revision 43444)
@@ -0,0 +1,106 @@
+package eu.dnetlib.msro.workflows.nodes;
+
+import java.io.StringReader;
+import java.util.Map;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.resultset.client.ResultSetClient;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+import eu.dnetlib.rmi.common.ResultSet;
+import eu.dnetlib.rmi.enabling.ISRegistryService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.io.SAXReader;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class UpdateRepositoryProfilesJobNode extends SimpleJobNode {
+
+	private static final String REPOSITORY_SERVICE_RESOURCE_TYPE = "RepositoryServiceResourceType";
+	private static final Log log = LogFactory.getLog(UpdateRepositoryProfilesJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+	@Autowired
+	private ResultSetClient resultSetClient;
+	@Autowired
+	private UniqueServiceLocator serviceLocator;
+	private String eprParam = "repoEpr";
+	private String existingReposParam = "existingRepos";
+
+	@Override
+	protected void beforeStart(final Token token) {
+		final ResultSet<?> epr = token.getEnv().getAttribute(getEprParam(), ResultSet.class);
+		token.setProgressProvider(new ResultsetProgressProvider(epr, this.resultSetClient));
+	}
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+		// datasourceID -> profileID
+		@SuppressWarnings("unchecked")
+		final Map<String, String> existingRepos = env.getAttribute(getExistingReposParam(), Map.class);
+		final ResultSet<?> epr = env.getAttribute(getEprParam(), ResultSet.class);
+
+		final SAXReader reader = new SAXReader();
+
+		int countUpdated = 0;
+		int countInserted = 0;
+		int countProfileErrors = 0;
+
+		log.info("Adding/updating repository profiles...");
+
+		final ISRegistryService registry = this.serviceLocator.getService(ISRegistryService.class);
+
+		for (final String profile : this.resultSetClient.iter(epr, String.class)) {
+			try {
+				final Document doc = reader.read(new StringReader(profile));
+				final String dsId = doc.valueOf("//EXTRA_FIELDS/FIELD[./key='OpenAireDataSourceId']/value");
+
+				log.debug("Registering/updating profile:\n " + profile + "\n");
+
+				if (existingRepos.containsKey(dsId)) {
+					final String profId = existingRepos.get(dsId);
+					doc.selectSingleNode("//RESOURCE_IDENTIFIER/@value").setText(profId);
+					registry.updateProfile(profId, doc.asXML(), REPOSITORY_SERVICE_RESOURCE_TYPE);
+					log.debug("Profile " + profId + " UPDATED for ds " + dsId);
+					countUpdated++;
+				} else {
+					final String profId = registry.registerProfile(doc.asXML());
+					log.debug("Valid Profile " + profId + " REGISTERED for ds " + dsId);
+					countInserted++;
+				}
+			} catch (final Exception e) {
+				log.error("INVALID PROFILE: " + profile, e);
+				countProfileErrors++;
+			}
+		}
+
+		log.info("   - updated_profiles: " + countUpdated);
+		log.info("   - inserted_profiles: " + countInserted);
+		log.info("   - profiles_with_errors: " + countProfileErrors);
+
+		env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "updated_profiles", countUpdated);
+		env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "inserted_profiles", countInserted);
+		env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "profiles_with_errors", countProfileErrors);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getEprParam() {
+		return this.eprParam;
+	}
+
+	public void setEprParam(final String eprParam) {
+		this.eprParam = eprParam;
+	}
+
+	public String getExistingReposParam() {
+		return this.existingReposParam;
+	}
+
+	public void setExistingReposParam(final String existingReposParam) {
+		this.existingReposParam = existingReposParam;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/repohi/RetrieveInterfaceInfoJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/repohi/RetrieveInterfaceInfoJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/repohi/RetrieveInterfaceInfoJobNode.java	(revision 43444)
@@ -0,0 +1,41 @@
+package eu.dnetlib.msro.workflows.nodes.repohi;
+
+import java.io.StringReader;
+import javax.annotation.Resource;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import org.dom4j.Document;
+import org.dom4j.Node;
+import org.dom4j.io.SAXReader;
+
+public class RetrieveInterfaceInfoJobNode extends SimpleJobNode {
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		final String datasourceId = env.getAttribute(WorkflowsConstants.LOG_DATASOURCE_ID, String.class);
+		final String interfaceID = env.getAttribute(WorkflowsConstants.LOG_DATASOURCE_INTERFACE, String.class);
+		final String profile = this.serviceLocator.getService(ISLookUpService.class).getResourceProfile(datasourceId);
+		final Document doc = new SAXReader().read(new StringReader(profile));
+		final Node ifcNode = doc.selectSingleNode("//INTERFACE[@id='" + interfaceID + "']");
+
+		final String contentDescription = ifcNode.valueOf("./@contentDescription");
+		env.setAttribute("objectStoreContentDescription", contentDescription);
+
+		final Node acProtNode = doc.selectSingleNode("//INTERFACE[@id='" + interfaceID + "']/ACCESS_PROTOCOL");
+
+		final String basePath = acProtNode.valueOf("./@basePath");
+		env.setAttribute("objectStoreBasePath", basePath);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/claims/DecapsuleClaimsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/claims/DecapsuleClaimsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/claims/DecapsuleClaimsJobNode.java	(revision 43444)
@@ -0,0 +1,95 @@
+package eu.dnetlib.msro.workflows.nodes.claims;
+
+import java.io.StringReader;
+import java.util.function.UnaryOperator;
+
+import eu.dnetlib.enabling.resultset.factory.ResultSetFactory;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.nodes.hadoop.SetHdfsFileJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.common.ResultSet;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.Element;
+import org.dom4j.Namespace;
+import org.dom4j.QName;
+import org.dom4j.io.SAXReader;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class DecapsuleClaimsJobNode extends SimpleJobNode {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(SetHdfsFileJobNode.class);
+
+	private static final Namespace OAF_NS = new Namespace("oaf", "http://namespace.openaire.eu/oaf");
+	private static final double DEFAULT_TRUST = 0.9;
+
+	private String inputEprParam;
+	private String outputEprParam;
+
+	@Autowired
+	private ResultSetFactory resultSetFactory;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+		final SAXReader reader = new SAXReader();
+
+		final UnaryOperator<String> mapper = row -> {
+			try {
+				final Document docRow = reader.read(new StringReader(row));
+
+				final String xml = docRow.valueOf("//ROW/FIELD[@name = 'xml']");
+				final String provenance = docRow.valueOf("//ROW/FIELD[@name = 'provenance']");
+
+				final Document doc = reader.read(new StringReader(xml));
+				final Element record = (Element) doc.selectSingleNode("//*[local-name() = 'record']");
+				final Element about = record.addElement(new QName("about"));
+				final Element dataInfo = about.addElement(new QName("datainfo", OAF_NS));
+
+				dataInfo.addElement(new QName("inferred", OAF_NS)).setText("false");
+				dataInfo.addElement(new QName("deletedbyinference", OAF_NS)).setText("false");
+				dataInfo.addElement(new QName("trust", OAF_NS)).setText(Double.toString(DEFAULT_TRUST));
+				dataInfo.addElement(new QName("inferenceprovenance", OAF_NS)).setText("");
+
+				final Element action = dataInfo.addElement(new QName("provenanceaction", OAF_NS));
+				action.addAttribute("schemename", "dnet:provenanceActions");
+				action.addAttribute("schemeid", "dnet:provenanceActions");
+				action.addAttribute("classname", provenance);
+				action.addAttribute("classid", provenance);
+
+				return doc.asXML();
+			} catch (final Throwable e) {
+				log.error("Error parsing row: " + row, e);
+				throw new IllegalArgumentException("Error parsing row: " + row, e);
+			}
+		};
+
+		final ResultSet<?> inputEpr = env.getAttribute(getInputEprParam(), ResultSet.class);
+		final ResultSet<String> epr = this.resultSetFactory.map(inputEpr, String.class, mapper);
+
+		env.setAttribute(getOutputEprParam(), epr);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getInputEprParam() {
+		return this.inputEprParam;
+	}
+
+	public void setInputEprParam(final String inputEprParam) {
+		this.inputEprParam = inputEprParam;
+	}
+
+	public String getOutputEprParam() {
+		return this.outputEprParam;
+	}
+
+	public void setOutputEprParam(final String outputEprParam) {
+		this.outputEprParam = outputEprParam;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/PromoteActionsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/PromoteActionsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/PromoteActionsJobNode.java	(revision 43444)
@@ -0,0 +1,38 @@
+package eu.dnetlib.msro.workflows.nodes.actionmanager;
+
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.rmi.data.hadoop.actionmanager.ActionManagerService;
+import org.apache.commons.lang3.StringUtils;
+
+public class PromoteActionsJobNode extends BlackboardJobNode {
+
+	public static final String ALL_SETS = "ALL SETS";
+
+	private String set;
+
+	@Override
+	protected String obtainServiceId(final Env env) {
+		return getServiceLocator().getServiceId(ActionManagerService.class);
+	}
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		job.setAction("PROMOTE");
+		if (!StringUtils.isBlank(getSet()) && !getSet().equals(ALL_SETS)) {
+			job.getParameters().put("set", getSet());
+			token.getEnv().setAttribute("set", getSet());
+		}
+	}
+
+	public String getSet() {
+		return this.set;
+	}
+
+	public void setSet(final String set) {
+		this.set = set;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/GarbageSetsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/GarbageSetsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/GarbageSetsJobNode.java	(revision 43444)
@@ -0,0 +1,29 @@
+package eu.dnetlib.msro.workflows.nodes.actionmanager;
+
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.rmi.data.hadoop.actionmanager.ActionManagerService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class GarbageSetsJobNode extends BlackboardJobNode {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(GarbageSetsJobNode.class);
+
+	@Override
+	protected String obtainServiceId(final Env env) {
+		return getServiceLocator().getServiceId(ActionManagerService.class);
+	}
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		log.info("preparing garbage collection for Action sets Job");
+		job.setAction("GARBAGE");
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/GarbageSetsHDFSJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/GarbageSetsHDFSJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/GarbageSetsHDFSJobNode.java	(revision 43444)
@@ -0,0 +1,32 @@
+package eu.dnetlib.msro.workflows.nodes.actionmanager;
+
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.rmi.data.hadoop.actionmanager.ActionManagerService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Created by claudio on 15/04/16.
+ */
+public class GarbageSetsHDFSJobNode extends BlackboardJobNode {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(GarbageSetsJobNode.class);
+
+	@Override
+	protected String obtainServiceId(final Env env) {
+		return getServiceLocator().getServiceId(ActionManagerService.class);
+	}
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		log.info("preparing garbage collection for Action sets Job");
+		job.setAction("GARBAGE_HDFS");
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/provision/FindIndexJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/provision/FindIndexJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/provision/FindIndexJobNode.java	(revision 43444)
@@ -0,0 +1,118 @@
+package eu.dnetlib.msro.workflows.nodes.provision;
+
+import java.util.List;
+import javax.annotation.Resource;
+
+import com.google.common.collect.Iterables;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class FindIndexJobNode extends SimpleJobNode {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(FindIndexJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	/**
+	 * if non null, overrides format from env.
+	 */
+	private String mdFormat;
+
+	/**
+	 * if non null, overrides format from env.
+	 */
+	private String layout;
+
+	/**
+	 * if non null, overrides format from env.
+	 */
+	private String interpretation;
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public String execute(final Env env) {
+		final String envFormat = env.getAttribute("format", String.class);
+		final String envLayout = env.getAttribute("layout", String.class);
+		final String envInterpretation = env.getAttribute("interpretation", String.class);
+
+		final String format = handleOverride(env, "format", envFormat, getMdFormat());
+		final String layout = handleOverride(env, "layout", envLayout, getLayout());
+		final String interp = handleOverride(env, "interpretation", envInterpretation, getInterpretation());
+
+		String mdRef = format + "-" + layout + "-" + interp;
+		log.info("searching index for [" + mdRef + "]");
+
+		final String indexId = findIndex(format, layout, interp);
+		env.setAttribute("index_id", indexId);
+
+		if (indexId == null || indexId.isEmpty()) {
+			log.info("no index was found for [" + mdRef + "]");
+			return "notFound";
+		} else {
+			log.info("index found for [" + mdRef + "]: " + indexId);
+			return "found";
+		}
+	}
+
+	private String findIndex(final String format, final String layout, final String interp) {
+		final String xquery =
+				"for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='IndexDSResourceType']" + "return $x[.//METADATA_FORMAT = '" + format
+						+ "' and .//METADATA_FORMAT_LAYOUT='" + layout + "' and .//METADATA_FORMAT_INTERPRETATION = '" + interp
+						+ "']//RESOURCE_IDENTIFIER/@value/string()";
+		try {
+			log.info("xquery: " + xquery);
+			List<String> ids = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery);
+			log.info("found indexDS ids: " + ids);
+			if (ids == null || ids.isEmpty()) { return null; }
+			if (ids.size() > 1) {
+				throw new IllegalStateException("found more than one index of given format: " + format + ", layout: " + layout
+						+ ", interpretation: " + interp);
+			}
+			return Iterables.getOnlyElement(ids);
+		} catch (Exception e) {
+			return null;
+		}
+	}
+
+	private String handleOverride(final Env env, final String name, final String envParam, final String override) {
+		if (override != null) {
+			env.setAttribute(name, override);
+		}
+		return override != null ? override : envParam;
+	}
+
+	public String getMdFormat() {
+		return mdFormat;
+	}
+
+	public void setMdFormat(final String mdFormat) {
+		this.mdFormat = mdFormat;
+	}
+
+	public String getLayout() {
+		return layout;
+	}
+
+	public void setLayout(final String layout) {
+		this.layout = layout;
+	}
+
+	public String getInterpretation() {
+		return interpretation;
+	}
+
+	public void setInterpretation(final String interpretation) {
+		this.interpretation = interpretation;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/LoadContextsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/LoadContextsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/LoadContextsJobNode.java	(revision 43444)
@@ -0,0 +1,45 @@
+package eu.dnetlib.msro.workflows.nodes;
+
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.index.ContextLoader;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class LoadContextsJobNode extends SimpleJobNode {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(LoadContextsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	private String contextParam;
+
+	@Autowired
+	private ContextLoader contextLoader;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		log.info("start preparing job");
+
+		final String contexts = this.contextLoader.load();
+
+		if (StringUtils.isBlank(contexts)) { throw new MSROException("empty contexts"); }
+
+		env.setAttribute(getContextParam(), contexts);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getContextParam() {
+		return contextParam;
+	}
+
+	public void setContextParam(final String contextParam) {
+		this.contextParam = contextParam;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/SetHdfsFileJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/SetHdfsFileJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/SetHdfsFileJobNode.java	(revision 43444)
@@ -0,0 +1,60 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * The Class SetHdfsFileJobNode.
+ */
+public class SetHdfsFileJobNode extends SimpleJobNode {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(SetHdfsFileJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	/**
+	 * The input records path.
+	 */
+	private String hdfsPath;
+
+	/**
+	 * The input records path param.
+	 */
+	private String hdfsPathParam;
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see com.googlecode.sarasvati.mem.MemNode#execute(com.googlecode.sarasvati.Engine, com.googlecode.sarasvati.NodeToken)
+	 */
+	@Override
+	public String execute(final Env env) {
+
+		log.info("hdfs path: " + getHdfsPath());
+
+		env.setAttribute(getHdfsPathParam(), getHdfsPath());
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getHdfsPath() {
+		return this.hdfsPath;
+	}
+
+	public void setHdfsPath(final String hdfsPath) {
+		this.hdfsPath = hdfsPath;
+	}
+
+	public String getHdfsPathParam() {
+		return this.hdfsPathParam;
+	}
+
+	public void setHdfsPathParam(final String hdfsPathParam) {
+		this.hdfsPathParam = hdfsPathParam;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/vre/VREPostJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/vre/VREPostJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/vre/VREPostJobNode.java	(revision 43444)
@@ -0,0 +1,133 @@
+package eu.dnetlib.msro.workflows.nodes.vre;
+
+import java.net.URI;
+
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.BasicResponseHandler;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+
+/**
+ * Created by alessia on 24/03/16.
+ */
+public class VREPostJobNode extends SimpleJobNode {
+
+	private static final Log log = LogFactory.getLog(VREPostJobNode.class);
+	private String scheme;
+	private String host;
+	private String postPath;
+
+	// POST parameters
+	private String appToken;
+	private String previewTitle;
+	private String previewDescription;
+	private String text;
+	private String httpImageUrl;
+	private boolean enableNotification;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+		final CloseableHttpClient httpclient = HttpClients.createDefault();
+		try {
+			final URI postUri = new URIBuilder().setScheme(this.scheme).setHost(this.host).setPath(this.postPath).build();
+			final HttpPost postAction = new HttpPost(postUri);
+			final StringBuilder strBuilder = new StringBuilder();
+			strBuilder.append("token=").append(this.appToken).append("&previewtitle=").append(this.previewTitle)
+					.append("&previewdescription=").append(this.previewDescription).append("&text=").append(this.text)
+					.append("&httpimageurl=").append(this.httpImageUrl).append("&enablenotification=").append(this.enableNotification);
+			final String postBody = strBuilder.toString();
+			log.debug("POST body: " + postBody);
+			final StringEntity ent = new StringEntity(postBody, ContentType.APPLICATION_FORM_URLENCODED);
+
+			postAction.setEntity(ent);
+
+			final BasicResponseHandler responseHandler = new BasicResponseHandler();
+
+			httpclient.execute(postAction, responseHandler);
+			return Arc.DEFAULT_ARC;
+		} finally {
+			httpclient.close();
+		}
+
+	}
+
+	public String getScheme() {
+		return this.scheme;
+	}
+
+	public void setScheme(final String scheme) {
+		this.scheme = scheme;
+	}
+
+	public String getHost() {
+		return this.host;
+	}
+
+	public void setHost(final String host) {
+		this.host = host;
+	}
+
+	public String getPostPath() {
+		return this.postPath;
+	}
+
+	public void setPostPath(final String postPath) {
+		this.postPath = postPath;
+	}
+
+	public String getAppToken() {
+		return this.appToken;
+	}
+
+	public void setAppToken(final String token) {
+		this.appToken = token;
+	}
+
+	public String getPreviewTitle() {
+		return this.previewTitle;
+	}
+
+	public void setPreviewTitle(final String previewTitle) {
+		this.previewTitle = previewTitle;
+	}
+
+	public String getPreviewDescription() {
+		return this.previewDescription;
+	}
+
+	public void setPreviewDescription(final String previewDescription) {
+		this.previewDescription = previewDescription;
+	}
+
+	public String getText() {
+		return this.text;
+	}
+
+	public void setText(final String text) {
+		this.text = text;
+	}
+
+	public String getHttpImageUrl() {
+		return this.httpImageUrl;
+	}
+
+	public void setHttpImageUrl(final String httpImageUrl) {
+		this.httpImageUrl = httpImageUrl;
+	}
+
+	public boolean isEnableNotification() {
+		return this.enableNotification;
+	}
+
+	public void setEnableNotification(final boolean enablenotification) {
+		this.enableNotification = enablenotification;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/LoadEntityLinksJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/LoadEntityLinksJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/LoadEntityLinksJobNode.java	(revision 43444)
@@ -0,0 +1,45 @@
+package eu.dnetlib.msro.workflows.nodes;
+
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.index.EntityGrouperConfigurationLoader;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class LoadEntityLinksJobNode extends SimpleJobNode {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(LoadEntityLinksJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	private String entityLinksParam;
+
+	@Autowired
+	private EntityGrouperConfigurationLoader grouperConfLoader;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		log.info("loading entity link table");
+
+		final String links = this.grouperConfLoader.load();
+
+		if (StringUtils.isBlank(links)) { throw new MSROException("empty links table"); }
+
+		env.setAttribute(getEntityLinksParam(), links);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getEntityLinksParam() {
+		return entityLinksParam;
+	}
+
+	public void setEntityLinksParam(final String entityLinksParam) {
+		this.entityLinksParam = entityLinksParam;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/PromoteShadowCacheStatsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/PromoteShadowCacheStatsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/PromoteShadowCacheStatsJobNode.java	(revision 43444)
@@ -0,0 +1,30 @@
+package eu.dnetlib.msro.workflows.nodes.stats;
+
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.procs.Token;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Ask the StatsManagerService to promote the shadow cache so that its entries can be used by the portal specified by the additional BB
+ * parameter <code>StatsManagerServiceBBAction.PROMOTE_SHADOW_CACHE.getTargetPortalParamName()</code>.
+ *
+ * @author alessia
+ */
+public class PromoteShadowCacheStatsJobNode extends AbstractStatsJobNode {
+
+	private static final Log log = LogFactory.getLog(PromoteShadowCacheStatsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		job.setAction(StatsManagerServiceBBAction.PROMOTE_SHADOW_CACHE.action());
+		final String portal = getPortalName(token.getEnv());
+		if (StringUtils.isNotBlank(portal)) {
+			job.getParameters().put(StatsManagerServiceBBAction.PROMOTE_SHADOW_CACHE.getTargetPortalParamName(), portal);
+		} else {
+			log.warn(StatsManagerServiceBBAction.PROMOTE_SHADOW_CACHE.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");
+		}
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/AbstractStatsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/AbstractStatsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/AbstractStatsJobNode.java	(revision 43444)
@@ -0,0 +1,75 @@
+package eu.dnetlib.msro.workflows.nodes.stats;
+
+import java.util.List;
+
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * Abstract BBJob node for sending BB messages to the StatsManagerService. Subclasses must implement the prepareJob method.
+ *
+ * @author alessia
+ */
+public abstract class AbstractStatsJobNode extends BlackboardJobNode {
+
+	private String xqueryForServiceIdParam;
+	/**
+	 * BB parameter defining which is the portal affected by the action. Overrides the value in the env parameter below.
+	 */
+	private String portal;
+	/**
+	 * Name of the env parameter where to find the target portal value. It is used only if the portal parameter above is not explicitly set
+	 * in the workflow.
+	 */
+	private String portalParam;
+
+	@Override
+	protected String obtainServiceId(final Env env) {
+		final String xquery = env.getAttribute(getXqueryForServiceIdParam(), String.class);
+		List<String> statsServiceIds;
+		try {
+			statsServiceIds = getServiceLocator().getService(ISLookUpService.class).quickSearchProfile(xquery);
+			if (statsServiceIds.size() > 1) { throw new RuntimeException("Too many StatsManagerService ids found using query: " + xquery); }
+			if (statsServiceIds.size() < 1) { throw new RuntimeException("StatsManagerService id not found using query: " + xquery); }
+			return statsServiceIds.get(0);
+		} catch (final ISLookUpException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	public String getPortalName(final Env env) {
+		if (StringUtils.isNotBlank(this.portal)) {
+			return this.portal;
+		} else {
+			return env.getAttribute(this.portalParam, String.class);
+		}
+	}
+
+	public String getXqueryForServiceIdParam() {
+		return this.xqueryForServiceIdParam;
+	}
+
+	public void setXqueryForServiceIdParam(final String xqueryForServiceIdParam) {
+		this.xqueryForServiceIdParam = xqueryForServiceIdParam;
+	}
+
+	public String getPortal() {
+		return this.portal;
+	}
+
+	public void setPortal(final String portal) {
+		this.portal = portal;
+	}
+
+	public String getPortalParam() {
+		return this.portalParam;
+	}
+
+	public void setPortalParam(final String portalParam) {
+		this.portalParam = portalParam;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/RefreshShadowCacheJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/RefreshShadowCacheJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/RefreshShadowCacheJobNode.java	(revision 43444)
@@ -0,0 +1,34 @@
+package eu.dnetlib.msro.workflows.nodes.stats;
+
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.procs.Token;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Ask the StatsManagerService to refresh the shadow cache according to the content of the shadow stats db for the portal specified by the
+ * additional BB parameter <code>StatsManagerServiceBBAction.REFRESH_SHADOW_CACHE.getTargetPortalParamName()</code>.
+ * <p>
+ * In order for the given portal to actually use the generated cache, we must promote the cache with the BB message 'promoteCache'.
+ * </p>
+ *
+ * @author alessia
+ * @see eu.dnetlib.msro.workflows.nodes.stats.PromoteShadowCacheStatsJobNode
+ */
+public class RefreshShadowCacheJobNode extends AbstractStatsJobNode {
+
+	private static final Log log = LogFactory.getLog(RefreshShadowCacheJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		job.setAction(StatsManagerServiceBBAction.REFRESH_SHADOW_CACHE.action());
+		final String portal = getPortalName(token.getEnv());
+		if (StringUtils.isNotBlank(portal)) {
+			job.getParameters().put(StatsManagerServiceBBAction.REFRESH_SHADOW_CACHE.getTargetPortalParamName(), portal);
+		} else {
+			log.warn(StatsManagerServiceBBAction.REFRESH_SHADOW_CACHE.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");
+		}
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/FetchRelClassesJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/FetchRelClassesJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/FetchRelClassesJobNode.java	(revision 43444)
@@ -0,0 +1,60 @@
+package eu.dnetlib.msro.workflows.nodes;
+
+import java.util.List;
+import javax.annotation.Resource;
+
+import com.google.gson.Gson;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class FetchRelClassesJobNode extends SimpleJobNode {
+
+	private static final Log log = LogFactory.getLog(FetchRelClassesJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	private String xquery;
+
+	private String relClassesName;
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		final String relClasses = new Gson().toJson(fetchRelClasses());
+
+		log.info("loaded relClasses: " + relClasses);
+
+		env.setAttribute(getRelClassesName(), relClasses);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	private List<String> fetchRelClasses() throws ISLookUpException {
+
+		log.info("Loading relClasses with xquery: " + getXquery());
+
+		return serviceLocator.getService(ISLookUpService.class).quickSearchProfile(getXquery());
+	}
+
+	public String getRelClassesName() {
+		return relClassesName;
+	}
+
+	public void setRelClassesName(final String relClassesName) {
+		this.relClassesName = relClassesName;
+	}
+
+	public String getXquery() {
+		return xquery;
+	}
+
+	public void setXquery(final String xquery) {
+		this.xquery = xquery;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/GetHBaseTableDescriptionJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/GetHBaseTableDescriptionJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/GetHBaseTableDescriptionJobNode.java	(revision 43444)
@@ -0,0 +1,27 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import java.util.List;
+
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.data.hadoop.HadoopService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class GetHBaseTableDescriptionJobNode extends AbstractHBaseAdminJobNode {
+
+	private static final Log log = LogFactory.getLog(GetHBaseTableDescriptionJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		log.info("getting table description: '" + getTableName() + "' on cluster: '" + getCluster() + "'");
+
+		final List<String> columns = getServiceLocator().getService(HadoopService.class).describeHbaseTable(getCluster(), getTableName());
+		log.debug(String.format("table '%s': " + columns, getTableName()));
+		env.setAttribute(getTableColumnsParamName(), asCSV(columns));
+
+		return Arc.DEFAULT_ARC;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/LoadMappingProfileJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/LoadMappingProfileJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/LoadMappingProfileJobNode.java	(revision 43444)
@@ -0,0 +1,61 @@
+package eu.dnetlib.msro.workflows.nodes;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+/**
+ * Created by claudio on 20/06/16.
+ */
+public class LoadMappingProfileJobNode extends SimpleJobNode {
+
+	private static final Log log = LogFactory.getLog(LoadMappingProfileJobNode.class);
+
+	@Autowired
+	private UniqueServiceLocator serviceLocator;
+
+	private String profileId;
+
+	private String loadedMappingParam;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		log.info(String.format("loading mapping from: %s", profileId));
+
+		final String xquery = String.format("/RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value='%s']/BODY/CONFIGURATION/SCRIPT/CODE/*[local-name()='stylesheet']",
+				getProfileId());
+		final String xslt = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xquery);
+
+		log.debug(String.format("mapping: %s", xslt));
+
+		if (StringUtils.isBlank(xslt)) {
+			throw new IllegalStateException("unable to load mapping from profile: " + getProfileId());
+		}
+
+		env.setAttribute(getLoadedMappingParam(), xslt);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getProfileId() {
+		return profileId;
+	}
+
+	public void setProfileId(final String profileId) {
+		this.profileId = profileId;
+	}
+
+	public String getLoadedMappingParam() {
+		return loadedMappingParam;
+	}
+
+	public void setLoadedMappingParam(final String loadedMappingParam) {
+		this.loadedMappingParam = loadedMappingParam;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/DefineHBaseOpenaireSchemaJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/DefineHBaseOpenaireSchemaJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/DefineHBaseOpenaireSchemaJobNode.java	(revision 43444)
@@ -0,0 +1,25 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import java.util.Set;
+
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.openaire.hadoop.utils.HBaseTableUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class DefineHBaseOpenaireSchemaJobNode extends AbstractHBaseAdminJobNode {
+
+	private static final Log log = LogFactory.getLog(DefineHBaseOpenaireSchemaJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		final Set<String> columns = HBaseTableUtils.listAllColumns();
+		log.info("table definition: " + columns);
+		env.setAttribute(getTableColumnsParamName(), asCSV(columns));
+
+		return Arc.DEFAULT_ARC;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/ValidateShadowStatsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/ValidateShadowStatsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/ValidateShadowStatsJobNode.java	(revision 43444)
@@ -0,0 +1,35 @@
+package eu.dnetlib.msro.workflows.nodes.stats;
+
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.procs.Token;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Ask the StatsManagerService to perform the content validation between the shadow stats db and the preproduction index (using the
+ * prepublic search service) and creates a report, which is serialized in the blackboard job parameters.
+ * <p>
+ * The report is relative to the shadow stats db related to the portal specified by the additional BB parameter
+ * <code>StatsManagerServiceBBAction.VALIDATE_SHADOW_STATS.getTargetPortalParamName()</code>
+ * </p>
+ *
+ * @author alessia
+ */
+public class ValidateShadowStatsJobNode extends AbstractStatsJobNode {
+
+	private static final Log log = LogFactory.getLog(ValidateShadowStatsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		job.setAction(StatsManagerServiceBBAction.VALIDATE_SHADOW_STATS.action());
+		final String portal = getPortalName(token.getEnv());
+		if (StringUtils.isNotBlank(portal)) {
+			job.getParameters().put(StatsManagerServiceBBAction.VALIDATE_SHADOW_STATS.getTargetPortalParamName(), portal);
+		} else {
+			log.warn(StatsManagerServiceBBAction.VALIDATE_SHADOW_STATS.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");
+		}
+
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/ExistHBaseTableJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/ExistHBaseTableJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/ExistHBaseTableJobNode.java	(revision 43444)
@@ -0,0 +1,30 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.data.hadoop.HadoopService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class ExistHBaseTableJobNode extends AbstractHBaseAdminJobNode {
+
+	private static final Log log = LogFactory.getLog(ExistHBaseTableJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		log.info("checking table existance: '" + getTableName() + "' on cluster: '" + getCluster() + "'");
+
+		final HadoopService hadoopService = getServiceLocator().getService(HadoopService.class);
+		boolean exists = hadoopService.existHbaseTable(getCluster(), getTableName());
+
+		log.info("table '" + getTableName() + "' exists: " + exists);
+
+		if (exists) {
+			final String tableDesc = hadoopService.describeHBaseTableConfiguration(getCluster(), getTableName());
+			env.setAttribute(getTableConfigurationParamName(), tableDesc);
+		}
+
+		return exists ? "drop" : "define";
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/PrepareIISMainParamsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/PrepareIISMainParamsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/PrepareIISMainParamsJobNode.java	(revision 43444)
@@ -0,0 +1,52 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import java.util.List;
+import java.util.Map;
+
+import com.google.gson.Gson;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.procs.Env;
+import org.springframework.beans.factory.annotation.Required;
+
+public class PrepareIISMainParamsJobNode extends PrepareIISParams {
+
+	private String xqueryObjectStores;
+
+	private String objectStoreParam = "import_content_objectstores_csv";
+
+	// Enable/Disable inference modules
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		super.prepare(env);
+
+		env.setAttribute(getObjectStoreParam(), getFilteredObjectStoreCSV(getXqueryObjectStores()));
+
+		@SuppressWarnings("unchecked")
+		final List<Map<String, String>> sets = new Gson().fromJson(env.getAttribute("sets", String.class), List.class);
+		for (final Map<String, String> set : sets) {
+			env.setAttribute(set.get("enablingProperty"), set.get("enabled"));
+		}
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getXqueryObjectStores() {
+		return this.xqueryObjectStores;
+	}
+
+	@Required
+	public void setXqueryObjectStores(final String xqueryObjectStores) {
+		this.xqueryObjectStores = xqueryObjectStores;
+	}
+
+	public String getObjectStoreParam() {
+		return this.objectStoreParam;
+	}
+
+	public void setObjectStoreParam(final String objectStoreParam) {
+		this.objectStoreParam = objectStoreParam;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/FindStatsServiceJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/FindStatsServiceJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/FindStatsServiceJobNode.java	(revision 43444)
@@ -0,0 +1,71 @@
+package eu.dnetlib.msro.workflows.nodes.stats;
+
+import java.util.List;
+import javax.annotation.Resource;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Value;
+
+public class FindStatsServiceJobNode extends SimpleJobNode {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(FindStatsServiceJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	@Value(value = "${dnet.openaire.service.stats.lookup.xquery}")
+	private String xquery;
+
+	private String xqueryForServiceIdParam;
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @throws ISLookUpException
+	 * @throws MSROException
+	 * @see com.googlecode.sarasvati.mem.MemNode#execute(com.googlecode.sarasvati.Engine, com.googlecode.sarasvati.NodeToken)
+	 */
+	@Override
+	public String execute(final Env env) throws ISLookUpException, MSROException {
+
+		log.info("lookup for stats service: " + getXquery());
+
+		final List<String> statsServiceIds = this.serviceLocator.getService(ISLookUpService.class).quickSearchProfile(getXquery());
+
+		if (statsServiceIds.size() > 1) { throw new MSROException("too many stats services: " + statsServiceIds); }
+
+		if (statsServiceIds.isEmpty()) {
+			return "notFound";
+		} else {
+			env.setAttribute(getXqueryForServiceIdParam(), getXquery());
+			return "found";
+		}
+	}
+
+	public String getXquery() {
+		return this.xquery;
+	}
+
+	public void setXquery(final String xquery) {
+		this.xquery = xquery;
+	}
+
+	public String getXqueryForServiceIdParam() {
+		return this.xqueryForServiceIdParam;
+	}
+
+	public void setXqueryForServiceIdParam(final String xqueryForServiceIdParam) {
+		this.xqueryForServiceIdParam = xqueryForServiceIdParam;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/RestoreStatsCacheJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/RestoreStatsCacheJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/RestoreStatsCacheJobNode.java	(revision 43444)
@@ -0,0 +1,34 @@
+package eu.dnetlib.msro.workflows.nodes.stats;
+
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.procs.Token;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Ask the StatsManagerService to restore the cache of the portal specified by the additional BB parameter
+ * <code>StatsManagerServiceBBAction.RESTORE_CACHE.getTargetPortalParamName()</code> to the last available backup.
+ * <p>
+ * To create a backup you must send a "backup" BB message.
+ * </p>
+ *
+ * @author alessia
+ * @see BackupStatsCacheJobNode
+ */
+public class RestoreStatsCacheJobNode extends AbstractStatsJobNode {
+
+	private static final Log log = LogFactory.getLog(RestoreStatsCacheJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		job.setAction(StatsManagerServiceBBAction.RESTORE_CACHE.action());
+		final String portal = getPortalName(token.getEnv());
+		if (StringUtils.isNotBlank(portal)) {
+			job.getParameters().put(StatsManagerServiceBBAction.RESTORE_CACHE.getTargetPortalParamName(), portal);
+		} else {
+			log.warn(StatsManagerServiceBBAction.RESTORE_CACHE.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");
+		}
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/MigrateStatsCacheJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/MigrateStatsCacheJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/MigrateStatsCacheJobNode.java	(revision 43444)
@@ -0,0 +1,51 @@
+package eu.dnetlib.msro.workflows.nodes.stats;
+
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Ask the StatsManagerService to copy a cache of a portal to another portal cache.
+ * <p>
+ * The source portal is specified in the BB parameter <code>StatsManagerServiceBBAction.MIGRATE_CACHE.getSourcePortalParamName()</code>.
+ * </p>
+ * <p>
+ * The target portal is specified in the BB parameter <code>StatsManagerServiceBBAction.MIGRATE_CACHE.getTargetPortalParamName()</code>
+ * </p>
+ *
+ * @author alessia
+ */
+public class MigrateStatsCacheJobNode extends AbstractStatsJobNode {
+
+	private static final Log log = LogFactory.getLog(MigrateStatsCacheJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	/**
+	 * BB parameter defining which is the portal with the source cache to migrate.
+	 */
+	private String sourcePortal;
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		job.setAction(StatsManagerServiceBBAction.MIGRATE_CACHE.action());
+		if (StringUtils.isBlank(this.sourcePortal)) { throw new MSROException("Parameter sourcePortal required to migrate the cache"); }
+		job.getParameters().put(StatsManagerServiceBBAction.MIGRATE_CACHE.getSourcePortalParamName(), this.sourcePortal);
+		final String targetPortal = getPortalName(token.getEnv());
+		if (StringUtils.isNotBlank(targetPortal)) {
+			job.getParameters().put(StatsManagerServiceBBAction.MIGRATE_CACHE.getTargetPortalParamName(), targetPortal);
+		} else {
+			log.warn(StatsManagerServiceBBAction.MIGRATE_CACHE.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");
+		}
+
+	}
+
+	public String getSourcePortal() {
+		return this.sourcePortal;
+	}
+
+	public void setSourcePortal(final String sourcePortal) {
+		this.sourcePortal = sourcePortal;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/StoreHBaseRecordsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/StoreHBaseRecordsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/StoreHBaseRecordsJobNode.java	(revision 43444)
@@ -0,0 +1,11 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import eu.dnetlib.rmi.data.hadoop.HadoopBlackboardActions;
+
+public class StoreHBaseRecordsJobNode extends AbstractHBaseJobNode {
+
+	@Override
+	protected HadoopBlackboardActions getAction() {
+		return HadoopBlackboardActions.IMPORT_EPR_HBASE;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/utils/Similarity.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/utils/Similarity.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/utils/Similarity.java	(revision 43444)
@@ -0,0 +1,38 @@
+package eu.dnetlib.msro.workflows.nodes.utils;
+
+import com.google.gson.Gson;
+import eu.dnetlib.data.proto.TypeProtos.Type;
+import eu.dnetlib.miscutils.collections.Pair;
+
+public class Similarity {
+
+	private Pair<String, String> pair;
+	private Type type;
+
+	public Similarity(final Pair<String, String> pair, final Type type) {
+		super();
+		this.setPair(pair);
+		this.setType(type);
+	}
+
+	public Pair<String, String> getPair() {
+		return pair;
+	}
+
+	public void setPair(final Pair<String, String> pair) {
+		this.pair = pair;
+	}
+
+	public Type getType() {
+		return type;
+	}
+
+	public void setType(final Type type) {
+		this.type = type;
+	}
+
+	@Override
+	public String toString() {
+		return new Gson().toJson(this, Similarity.class);
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/PrepareIISParams.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/PrepareIISParams.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/PrepareIISParams.java	(revision 43444)
@@ -0,0 +1,301 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import javax.annotation.Resource;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.data.hadoop.ClusterName;
+import eu.dnetlib.rmi.data.hadoop.HadoopService;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+
+public abstract class PrepareIISParams extends SimpleJobNode {
+
+	private static final Log log = LogFactory.getLog(PrepareIISParams.class);
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	private String clusterName;
+
+	private String clusterParam = "cluster";
+
+	private String zkQuorumParam = "export_action_hbase_remote_zookeeper_quorum";
+
+	private String zkPortParam = "export_action_hbase_remote_zookeeper_clientport";
+
+	private String oozieWfAppPath;
+
+	private String oozieWfAppPathParam = "oozie.wf.application.path";
+
+	private String xqueryMdStoreService;
+
+	private String mdStoreStoreLocationParam = "import_mdstore_service_location";
+
+	private String xqueryObjectStoreService;
+
+	private String objectStoreLocationParam = "import_content_object_store_location";
+
+	private String xqueryIsLookupService;
+
+	private String islookupLocationParam = "import_islookup_service_location";
+
+	private String importProjectConceptsContextIdParam = "import_project_concepts_context_id";
+
+	private String importProjectConceptsContextId;
+
+	private String xqueryDatasetStore;
+
+	private String mdStoreDatasetParam = "import_dataset_mdstore_ids_csv";
+
+	private String objectStoreBlacklistCSV = "";
+
+	protected void prepare(final Env env) throws Exception {
+
+		env.setAttribute(getClusterParam(), getClusterName());
+
+		// Assumes we only have one mdStore service instance
+		env.setAttribute(getMdStoreStoreLocationParam(), getServiceEndpoint(getXqueryMdStoreService()));
+		// Assumes we only have one objectStore service instance
+		env.setAttribute(getObjectStoreLocationParam(), getServiceEndpoint(getXqueryObjectStoreService()));
+
+		env.setAttribute(getIslookupLocationParam(), getServiceEndpoint(getXqueryIsLookupService()));
+
+		env.setAttribute(getImportProjectConceptsContextIdParam(), getImportProjectConceptsContextId());
+
+		final Map<String, String> conf = serviceLocator.getService(HadoopService.class).getClusterConfiguration(getClusterName());
+
+		final Map<String, String> exportConf = serviceLocator.getService(HadoopService.class).getClusterConfiguration(ClusterName.DM.toString());
+
+		final String zkPort = exportConf.get("hbase.zookeeper.property.clientPort");
+		final String zkQuorum = getZkQuorumCSV(exportConf, zkPort);
+
+		env.setAttribute(getZkQuorumParam(), zkQuorum);
+		env.setAttribute(getZkPortParam(), zkPort);
+
+		final String nameNode = conf.get("fs.defaultFS");
+
+		env.setAttribute("nameNode", nameNode);
+		env.setAttribute("jobTracker", conf.get("mapred.job.tracker"));
+		env.setAttribute(getOozieWfAppPathParam(), getURI(nameNode, getOozieWfAppPath()));
+		env.setAttribute(getMdStoreDatasetParam(), asCSV(getProfileIds(getXqueryDatasetStore())));
+	}
+
+	protected String getServiceEndpoint(final String xquery) throws MSROException {
+		try {
+			return Iterables.getOnlyElement(this.serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery));
+		} catch (final ISLookUpException e) {
+			throw new MSROException("unable to fetch service endpoint", e);
+		} catch (final NoSuchElementException e) {
+			throw new MSROException("unable to find service endpoint, xquery: " + getXqueryMdStoreService(), e);
+		} catch (final IllegalArgumentException e) {
+			throw new MSROException("more than one services found, we assume to have only one available", e);
+		}
+	}
+
+	protected String getProfileId(final String xquery) throws MSROException {
+		try {
+			return Iterables.getOnlyElement(this.serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery));
+		} catch (final ISLookUpException e) {
+			throw new MSROException("unable to fetch profile id", e);
+		} catch (final NoSuchElementException e) {
+			throw new MSROException("unable to find profile profile, xquery: " + xquery, e);
+		} catch (final IllegalArgumentException e) {
+			throw new MSROException("more than one profile profiles was found, we assume to have only one available, xquery: " + xquery, e);
+		}
+	}
+
+	protected List<String> getProfileIds(final String xquery) throws MSROException {
+		try {
+			final List<String> ids = this.serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery);
+
+			if (ids.isEmpty()) {
+				log.warn("couldn't find any profile, xquery: " + xquery);
+			}
+
+			return ids;
+		} catch (final ISLookUpException e) {
+			throw new MSROException("unable to fetch profile ids, x query: " + xquery, e);
+		}
+	}
+
+	protected String getFilteredObjectStoreCSV(final String xquery) throws MSROException {
+		return asCSV(filter(getProfileIds(xquery), asList(getObjectStoreBlacklistCSV())));
+	}
+
+	protected List<String> filter(final List<String> list, final List<String> filter) {
+		if (filter == null || filter.isEmpty()) { return list; }
+		list.removeAll(filter);
+		return list;
+	}
+
+	protected String asCSV(final List<String> list) {
+		return Joiner.on(",").skipNulls().join(list);
+	}
+
+	protected List<String> asList(final String csv) {
+		return Lists.newArrayList(Splitter.on(",").trimResults().omitEmptyStrings().split(csv));
+	}
+
+	private String getURI(final String nameNode, final String relative) {
+		// TODO ensure to return a valid URI
+		return nameNode + relative;
+	}
+
+	private String getZkQuorumCSV(final Map<String, String> conf, final String zkPort) {
+		return Joiner.on(":" + zkPort + ",").join(Splitter.on(",").omitEmptyStrings().split(conf.get("hbase.zookeeper.quorum")));
+	}
+
+	public String getXqueryMdStoreService() {
+		return this.xqueryMdStoreService;
+	}
+
+	@Required
+	public void setXqueryMdStoreService(final String xqueryMdStoreService) {
+		this.xqueryMdStoreService = xqueryMdStoreService;
+	}
+
+	public String getMdStoreStoreLocationParam() {
+		return this.mdStoreStoreLocationParam;
+	}
+
+	public void setMdStoreStoreLocationParam(final String mdStoreStoreLocationParam) {
+		this.mdStoreStoreLocationParam = mdStoreStoreLocationParam;
+	}
+
+	public String getClusterName() {
+		return this.clusterName;
+	}
+
+	public void setClusterName(final String clusterName) {
+		this.clusterName = clusterName;
+	}
+
+	public String getZkQuorumParam() {
+		return this.zkQuorumParam;
+	}
+
+	public void setZkQuorumParam(final String zkQuorumParam) {
+		this.zkQuorumParam = zkQuorumParam;
+	}
+
+	public String getZkPortParam() {
+		return this.zkPortParam;
+	}
+
+	public void setZkPortParam(final String zkPortParam) {
+		this.zkPortParam = zkPortParam;
+	}
+
+	public String getClusterParam() {
+		return this.clusterParam;
+	}
+
+	public void setClusterParam(final String clusterParam) {
+		this.clusterParam = clusterParam;
+	}
+
+	public String getOozieWfAppPathParam() {
+		return this.oozieWfAppPathParam;
+	}
+
+	public void setOozieWfAppPathParam(final String oozieWfAppPathParam) {
+		this.oozieWfAppPathParam = oozieWfAppPathParam;
+	}
+
+	public String getOozieWfAppPath() {
+		return this.oozieWfAppPath;
+	}
+
+	public void setOozieWfAppPath(final String oozieWfAppPath) {
+		this.oozieWfAppPath = oozieWfAppPath;
+	}
+
+	@Required
+	public String getXqueryDatasetStore() {
+		return this.xqueryDatasetStore;
+	}
+
+	public void setXqueryDatasetStore(final String xqueryDatasetStore) {
+		this.xqueryDatasetStore = xqueryDatasetStore;
+	}
+
+	public String getMdStoreDatasetParam() {
+		return this.mdStoreDatasetParam;
+	}
+
+	public void setMdStoreDatasetParam(final String mdStoreDatasetParam) {
+		this.mdStoreDatasetParam = mdStoreDatasetParam;
+	}
+
+	public String getXqueryObjectStoreService() {
+		return this.xqueryObjectStoreService;
+	}
+
+	@Required
+	public void setXqueryObjectStoreService(final String xqueryObjectStoreService) {
+		this.xqueryObjectStoreService = xqueryObjectStoreService;
+	}
+
+	public String getObjectStoreLocationParam() {
+		return this.objectStoreLocationParam;
+	}
+
+	public void setObjectStoreLocationParam(final String objectStoreLocationParam) {
+		this.objectStoreLocationParam = objectStoreLocationParam;
+	}
+
+	public String getObjectStoreBlacklistCSV() {
+		return this.objectStoreBlacklistCSV;
+	}
+
+	public void setObjectStoreBlacklistCSV(final String objectStoreBlacklistCSV) {
+		this.objectStoreBlacklistCSV = objectStoreBlacklistCSV;
+	}
+
+	public String getXqueryIsLookupService() {
+		return this.xqueryIsLookupService;
+	}
+
+	@Required
+	public void setXqueryIsLookupService(final String xqueryIsLookupService) {
+		this.xqueryIsLookupService = xqueryIsLookupService;
+	}
+
+	public String getIslookupLocationParam() {
+		return this.islookupLocationParam;
+	}
+
+	public void setIslookupLocationParam(final String islookupLocationParam) {
+		this.islookupLocationParam = islookupLocationParam;
+	}
+
+	public String getImportProjectConceptsContextIdParam() {
+		return this.importProjectConceptsContextIdParam;
+	}
+
+	public void setImportProjectConceptsContextIdParam(final String importProjectConceptsContextIdParam) {
+		this.importProjectConceptsContextIdParam = importProjectConceptsContextIdParam;
+	}
+
+	public String getImportProjectConceptsContextId() {
+		return this.importProjectConceptsContextId;
+	}
+
+	public void setImportProjectConceptsContextId(final String importProjectConceptsContextId) {
+		this.importProjectConceptsContextId = importProjectConceptsContextId;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/utils/DedupConfigurationOrchestrationLoader.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/utils/DedupConfigurationOrchestrationLoader.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/utils/DedupConfigurationOrchestrationLoader.java	(revision 43444)
@@ -0,0 +1,110 @@
+package eu.dnetlib.msro.workflows.nodes.dedup.utils;
+
+import java.io.StringReader;
+import java.util.List;
+import java.util.Queue;
+import javax.annotation.Resource;
+
+import com.google.common.collect.Lists;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.pace.config.DedupConfig;
+import eu.dnetlib.rmi.enabling.ISLookUpDocumentNotFoundException;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.DocumentException;
+import org.dom4j.Element;
+import org.dom4j.io.SAXReader;
+
+/**
+ * The Class DedupConfigurationOrchestrationLoader.
+ */
+public class DedupConfigurationOrchestrationLoader {
+
+	/**
+	 * The Constant log.
+	 */
+	private static final Log log = LogFactory.getLog(DedupConfigurationOrchestrationLoader.class);
+
+	/**
+	 * The service locator.
+	 */
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	/**
+	 * Load the dedup orchestration profile from the IS.
+	 *
+	 * @param id the id
+	 * @return the dedup configuration orchestration
+	 * @throws ISLookUpDocumentNotFoundException the IS look up document not found exception
+	 * @throws ISLookUpException                 the IS look up exception
+	 * @throws DocumentException                 the document exception
+	 */
+	public DedupConfigurationOrchestration loadByActionSetId(final String id) throws Exception {
+
+		final ISLookUpService isLookUpService = serviceLocator.getService(ISLookUpService.class);
+
+		final String xquery = String.format("/RESOURCE_PROFILE[.//DEDUPLICATION/ACTION_SET/@id = '%s']", id);
+		log.info("loading dedup orchestration: " + xquery);
+
+		return parseOrchestrationProfile(isLookUpService, isLookUpService.getResourceProfileByQuery(xquery));
+	}
+
+	public List<DedupConfigurationOrchestration> loadByEntityName(final String entityName) throws Exception {
+
+		final ISLookUpService isLookUpService = serviceLocator.getService(ISLookUpService.class);
+
+		final String xquery = String.format("/RESOURCE_PROFILE[.//DEDUPLICATION/ENTITY/@name = '%s']", entityName);
+		log.info("loading dedup orchestration: " + xquery);
+
+		final List<DedupConfigurationOrchestration> res = Lists.newArrayList();
+
+		for (final String profile : isLookUpService.quickSearchProfile(xquery)) {
+			res.add(parseOrchestrationProfile(isLookUpService, profile));
+		}
+
+		return res;
+	}
+
+	private DedupConfigurationOrchestration parseOrchestrationProfile(final ISLookUpService isLookUpService, final String dedupOrchestation)
+			throws DocumentException, ISLookUpException {
+
+		final Document doc = new SAXReader().read(new StringReader(dedupOrchestation));
+
+		final Element e = (Element) doc.selectSingleNode("//DEDUPLICATION/ENTITY");
+		final Entity entity = new Entity(e.attributeValue("name"), e.attributeValue("code"), e.attributeValue("label"));
+
+		final String actionSetId = doc.valueOf("//DEDUPLICATION/ACTION_SET/@id");
+		final Queue<DedupConfig> configurations = Lists.newLinkedList();
+
+		for (final Object o : doc.selectNodes("//SCAN_SEQUENCE/SCAN")) {
+			configurations.add(loadConfig(isLookUpService, actionSetId, o));
+		}
+
+		final DedupConfigurationOrchestration dco = new DedupConfigurationOrchestration(entity, actionSetId, configurations);
+
+		log.debug("loaded dedup configuration orchestration: " + dco.toString());
+		log.info("loaded dedup configuration orchestration, size: " + dco.getConfigurations().size());
+		return dco;
+	}
+
+	private DedupConfig loadConfig(final ISLookUpService isLookUpService, final String actionSetId, final Object o) throws ISLookUpException {
+		final Element s = (Element) o;
+		final String configProfileId = s.attributeValue("id");
+		final String conf =
+				isLookUpService.getResourceProfileByQuery(String.format(
+						"for $x in /RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '%s'] return $x//DEDUPLICATION/text()",
+						configProfileId));
+		log.debug("loaded dedup configuration from IS profile: " + conf);
+		if (StringUtils.isBlank(conf)) {
+			throw new IllegalStateException(String.format("got empty dedup configuration profile for '%s'", configProfileId));
+		}
+		final DedupConfig dedupConfig = DedupConfig.load(conf);
+		dedupConfig.getWf().setConfigurationId(actionSetId);
+		return dedupConfig;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/PrepareIISPreprocessingParamsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/PrepareIISPreprocessingParamsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/PrepareIISPreprocessingParamsJobNode.java	(revision 43444)
@@ -0,0 +1,106 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.procs.Env;
+import org.springframework.beans.factory.annotation.Required;
+
+public class PrepareIISPreprocessingParamsJobNode extends PrepareIISParams {
+
+	private String xqueryWosMDStore;
+
+	private String mdStoreWosParam = "import_wos_mdstore_id";
+
+	private String xqueryDatabaseService;
+
+	private String databaseServiceLocationParam = "import_database_service_location";
+
+	private String xqueryDataciteObjectStore;
+
+	private String dataciteObjectStoreParam = "import_content_datacite_objectstores_csv";
+
+	private String xqueryWosObjectStore;
+
+	private String wosObjectStorePlaintextParam = "import_content_wos_plaintext_objectstores_csv";
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+		super.prepare(env);
+
+		env.setAttribute(getMdStoreWosParam(), getProfileId(getXqueryWosMDStore()));
+		env.setAttribute(getDatabaseServiceLocationParam(), getServiceEndpoint(getXqueryDatabaseService()));
+
+		env.setAttribute(getDataciteObjectStoreParam(), getFilteredObjectStoreCSV(getXqueryDataciteObjectStore()));
+		env.setAttribute(getWosObjectStorePlaintextParam(), getProfileId(getXqueryWosObjectStore()));
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getXqueryWosMDStore() {
+		return this.xqueryWosMDStore;
+	}
+
+	@Required
+	public void setXqueryWosMDStore(final String xqueryWosMDStore) {
+		this.xqueryWosMDStore = xqueryWosMDStore;
+	}
+
+	public String getMdStoreWosParam() {
+		return this.mdStoreWosParam;
+	}
+
+	public void setMdStoreWosParam(final String mdStoreWosParam) {
+		this.mdStoreWosParam = mdStoreWosParam;
+	}
+
+	public String getXqueryDatabaseService() {
+		return this.xqueryDatabaseService;
+	}
+
+	@Required
+	public void setXqueryDatabaseService(final String xqueryDatabaseService) {
+		this.xqueryDatabaseService = xqueryDatabaseService;
+	}
+
+	public String getDatabaseServiceLocationParam() {
+		return this.databaseServiceLocationParam;
+	}
+
+	public void setDatabaseServiceLocationParam(final String databaseServiceLocationParam) {
+		this.databaseServiceLocationParam = databaseServiceLocationParam;
+	}
+
+	public String getWosObjectStorePlaintextParam() {
+		return this.wosObjectStorePlaintextParam;
+	}
+
+	public void setWosObjectStorePlaintextParam(final String wosObjectStorePlaintextParam) {
+		this.wosObjectStorePlaintextParam = wosObjectStorePlaintextParam;
+	}
+
+	public String getXqueryWosObjectStore() {
+		return this.xqueryWosObjectStore;
+	}
+
+	@Required
+	public void setXqueryWosObjectStore(final String xqueryWosObjectStore) {
+		this.xqueryWosObjectStore = xqueryWosObjectStore;
+	}
+
+	public String getXqueryDataciteObjectStore() {
+		return this.xqueryDataciteObjectStore;
+	}
+
+	@Required
+	public void setXqueryDataciteObjectStore(final String xqueryDataciteObjectStore) {
+		this.xqueryDataciteObjectStore = xqueryDataciteObjectStore;
+	}
+
+	public String getDataciteObjectStoreParam() {
+		return this.dataciteObjectStoreParam;
+	}
+
+	public void setDataciteObjectStoreParam(final String dataciteObjectStoreParam) {
+		this.dataciteObjectStoreParam = dataciteObjectStoreParam;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/CheckHDFSCountJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/CheckHDFSCountJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/CheckHDFSCountJobNode.java	(revision 43444)
@@ -0,0 +1,75 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class CheckHDFSCountJobNode extends SimpleJobNode {
+
+	private static final Log log = LogFactory.getLog(CheckHDFSCountJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	/**
+	 * Name of the env param where to find the number to check.
+	 **/
+	private String numberToVerifyParamName;
+	/**
+	 * Name of the enc param where to find the number of records stored to hdfs.
+	 **/
+	private String hdfsCounterParamName = "mainlog:storeHdfsRecords:count";
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @throws MSROException
+	 * @see com.googlecode.sarasvati.mem.MemNode#execute(com.googlecode.sarasvati.Engine, com.googlecode.sarasvati.NodeToken)
+	 */
+	@Override
+	public String execute(final Env env) throws MSROException {
+		int hdfsCount = 0;
+		int hbaseCount = 0;
+		final String numberToVerifyInEnvParam = WorkflowsConstants.BLACKBOARD_PARAM_PREFIX + this.numberToVerifyParamName;
+		hbaseCount = getValueFromEnv(env, numberToVerifyInEnvParam);
+		hdfsCount = getValueFromEnv(env, this.hdfsCounterParamName);
+
+		if (hdfsCount != hbaseCount) {
+			log.warn("Numbers are not the same. Number to verify (hbase count): " + hbaseCount + ". From hdfs: " + hdfsCount);
+			log.warn((hdfsCount - hbaseCount) + " records come without a 'metadata' field");
+			env.setAttribute("noMetadataRecords", hdfsCount - hbaseCount);
+		}
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	private int getValueFromEnv(final Env env, final String paramName) throws MSROException {
+		final String valueString = env.getAttribute(paramName, String.class);
+		int number = 0;
+		try {
+			number = Integer.parseInt(valueString);
+		} catch (final NumberFormatException e) {
+			log.error("Env param name: " + paramName + " is " + valueString + " and cannot be parsed as integer");
+			throw new MSROException(e);
+		}
+		return number;
+	}
+
+	public String getNumberToVerifyParamName() {
+		return this.numberToVerifyParamName;
+	}
+
+	public void setNumberToVerifyParamName(final String numberToVerifyParamName) {
+		this.numberToVerifyParamName = numberToVerifyParamName;
+	}
+
+	public String getHdfsCounterParamName() {
+		return this.hdfsCounterParamName;
+	}
+
+	public void setHdfsCounterParamName(final String hdfsCounterParamName) {
+		this.hdfsCounterParamName = hdfsCounterParamName;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/DropHBaseTableJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/DropHBaseTableJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/DropHBaseTableJobNode.java	(revision 43444)
@@ -0,0 +1,34 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.data.hadoop.HadoopService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * The Class DropHBaseTableJobNode.
+ */
+public class DropHBaseTableJobNode extends AbstractHBaseAdminJobNode {
+
+	/**
+	 * The Constant log.
+	 */
+	private static final Log log = LogFactory.getLog(DropHBaseTableJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see eu.dnetlib.msro.workflows.nodes.SimpleJobNode#execute(com.googlecode.sarasvati.NodeToken)
+	 */
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		log.info("Dropping hbase table '" + getTableName() + "' on cluster: '" + getCluster() + "'");
+
+		getServiceLocator().getService(HadoopService.class).dropHbaseTable(getCluster(), getTableName());
+
+		return Arc.DEFAULT_ARC;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/datacite/CollectDatasetsByJournalJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/datacite/CollectDatasetsByJournalJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/datacite/CollectDatasetsByJournalJobNode.java	(revision 43444)
@@ -0,0 +1,153 @@
+package eu.dnetlib.msro.workflows.nodes.datacite;
+
+import java.io.StringReader;
+import java.util.Iterator;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+
+import eu.dnetlib.data.collector.plugins.datasets.DatasetsByJournalIterator;
+import eu.dnetlib.data.collector.plugins.datasets.PangaeaJournalInfo;
+import eu.dnetlib.enabling.resultset.client.ResultSetClient;
+import eu.dnetlib.enabling.resultset.factory.ResultSetFactory;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.common.ResultSet;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.io.SAXReader;
+import org.springframework.beans.factory.annotation.Autowired;
+
+// TODO: Auto-generated Javadoc
+
+/**
+ * The Class CollectDatasetsByJournalJobNode.
+ */
+public class CollectDatasetsByJournalJobNode extends SimpleJobNode {
+
+	private static final Log log = LogFactory.getLog(CollectDatasetsByJournalJobNode.class);
+
+	public static PangaeaJournalInfo END_QUEUE = new PangaeaJournalInfo();
+	/**
+	 * The executor.
+	 */
+	private final Executor executor = Executors.newSingleThreadExecutor();
+	/**
+	 * The datasource id.
+	 */
+	private String datasourceId;
+	/**
+	 * The output epr param.
+	 */
+	private String outputEprParam;
+	/**
+	 * The input epr param.
+	 */
+	private String inputEprParam;
+	/**
+	 * The result set client factory.
+	 */
+	@Autowired
+	private ResultSetClient resultSetClient;
+	/**
+	 * The result set factory.
+	 */
+	@Autowired
+	private ResultSetFactory resultSetFactory;
+
+	/**
+	 * (non-Javadoc)
+	 *
+	 * @see SimpleJobNode#execute(com.googlecode.sarasvati.NodeToken)
+	 */
+	@Override
+	protected String execute(final Env env) throws Exception {
+		final ResultSet<?> inputEpr = env.getAttribute(this.inputEprParam, ResultSet.class);
+		final Iterable<String> input = this.resultSetClient.iter(inputEpr, String.class);
+		final BlockingQueue<PangaeaJournalInfo> publicationsQueue = new ArrayBlockingQueue<PangaeaJournalInfo>(50);
+
+		this.executor.execute(() -> {
+			final SAXReader reader = new SAXReader();
+
+			for (final String inputString : input) {
+				try {
+					final Document doc = reader.read(new StringReader(inputString));
+					final String jId = doc.valueOf("//FIELD[@name='id']");
+					final String dsId = doc.valueOf("//FIELD[@name='datasource']");
+					final String dsName = doc.valueOf("//FIELD[@name='name']");
+					final String jISSN = doc.valueOf("//FIELD[@name='issn']");
+					final PangaeaJournalInfo info = new PangaeaJournalInfo();
+					info.setDatasourceId(dsId);
+					info.setJournalId(jId);
+					info.setJournalName(dsName);
+					info.setJournalISSN(jISSN);
+					publicationsQueue.put(info);
+				} catch (final Exception e) {
+					log.error(e);
+				}
+			}
+			try {
+				publicationsQueue.put(END_QUEUE);
+			} catch (final InterruptedException e) {
+				log.error(e);
+			}
+		});
+
+		final Iterator<PangaeaJournalInfo> itOnQueue =
+				new IteratorOnQueue<PangaeaJournalInfo>(publicationsQueue, CollectDatasetsByJournalJobNode.END_QUEUE);
+
+		final ResultSet<String> eprOutput = this.resultSetFactory.createResultSet(() -> new DatasetsByJournalIterator(itOnQueue));
+		env.setAttribute(getOutputEprParam(), eprOutput);
+		return Arc.DEFAULT_ARC;
+	}
+
+	/**
+	 * Gets the datasource id.
+	 *
+	 * @return the datasourceId
+	 */
+	public String getDatasourceId() {
+		return this.datasourceId;
+	}
+
+	/**
+	 * Sets the datasource id.
+	 *
+	 * @param datasourceId the datasourceId to set
+	 */
+	public void setDatasourceId(final String datasourceId) {
+		this.datasourceId = datasourceId;
+	}
+
+	/**
+	 * @return the outputEprParam
+	 */
+	public String getOutputEprParam() {
+		return this.outputEprParam;
+	}
+
+	/**
+	 * @param outputEprParam the outputEprParam to set
+	 */
+	public void setOutputEprParam(final String outputEprParam) {
+		this.outputEprParam = outputEprParam;
+	}
+
+	/**
+	 * @return the inputEprParam
+	 */
+	public String getInputEprParam() {
+		return this.inputEprParam;
+	}
+
+	/**
+	 * @param inputEprParam the inputEprParam to set
+	 */
+	public void setInputEprParam(final String inputEprParam) {
+		this.inputEprParam = inputEprParam;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/PromoteShadowStatsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/PromoteShadowStatsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/stats/PromoteShadowStatsJobNode.java	(revision 43444)
@@ -0,0 +1,40 @@
+package eu.dnetlib.msro.workflows.nodes.stats;
+
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.procs.Token;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Ask the StatsManagerService to promote the shadow stats db and cache to public for the portal specified by the additional BB parameter
+ * <code>StatsManagerServiceBBAction.PROMOTE_SHADOW_STATS.getTargetPortalParamName()</code>.
+ * <p>
+ * <p>
+ * According to the StatsManagerService specifications in https://issue.openaire.research-infrastructures.eu/issues/914#note-49, this BB
+ * action will:
+ * <ol>
+ * <li>promote the shadow cache to public</li>
+ * <li>make a backup of the current stats db schema under the name 'backup'</li>
+ * <li>replace it with the new one</li>
+ * </ol>
+ * </p>
+ *
+ * @author alessia
+ */
+public class PromoteShadowStatsJobNode extends AbstractStatsJobNode {
+
+	private static final Log log = LogFactory.getLog(PromoteShadowStatsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		job.setAction(StatsManagerServiceBBAction.PROMOTE_SHADOW_STATS.action());
+		final String portal = getPortalName(token.getEnv());
+		if (StringUtils.isNotBlank(portal)) {
+			job.getParameters().put(StatsManagerServiceBBAction.PROMOTE_SHADOW_STATS.getTargetPortalParamName(), portal);
+		} else {
+			log.warn(StatsManagerServiceBBAction.PROMOTE_SHADOW_STATS.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");
+		}
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/DeleteHBaseRecordsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/DeleteHBaseRecordsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/DeleteHBaseRecordsJobNode.java	(revision 43444)
@@ -0,0 +1,12 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import eu.dnetlib.rmi.data.hadoop.HadoopBlackboardActions;
+
+public class DeleteHBaseRecordsJobNode extends AbstractHBaseJobNode {
+
+	@Override
+	protected HadoopBlackboardActions getAction() {
+		return HadoopBlackboardActions.DELETE_EPR_HBASE;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/ObtainOpenaireDataSourceParamsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/ObtainOpenaireDataSourceParamsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/ObtainOpenaireDataSourceParamsJobNode.java	(revision 43444)
@@ -0,0 +1,50 @@
+package eu.dnetlib.msro.workflows.nodes;
+
+import javax.annotation.Resource;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+
+public class ObtainOpenaireDataSourceParamsJobNode extends SimpleJobNode {
+
+	private String providerId;
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+		final String query = "let $x := /*[.//RESOURCE_IDENTIFIER/@value='" + this.providerId + "']//EXTRA_FIELDS\n"
+				+ "return concat($x/FIELD[./key='OpenAireDataSourceId']/value, ' @@@ ', $x/FIELD[./key='NamespacePrefix']/value)";
+
+		final String[] arr = this.serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(query).split("@@@");
+
+		final String origId = arr[0].trim();
+		final String nsPrefix = arr[1].trim();
+		// this is needed by the mdbuilder
+		// TODO: update mdbuilder to use the env attributes below, whose names are defined in WorkflowConstants
+		env.setAttribute("parentDatasourceId", origId);
+		env.setAttribute("namespacePrefix", nsPrefix);
+		env.setAttribute("dateOfCollection", DateUtils.now_ISO8601());
+
+		// these are needed for validation and fill hostedby
+		env.setAttribute(WorkflowsConstants.LOG_DATASOURCE_ID, origId);
+		env.setAttribute(WorkflowsConstants.DATASOURCE_PREFIX, nsPrefix);
+
+		return Arc.DEFAULT_ARC;
+
+	}
+
+	public String getProviderId() {
+		return this.providerId;
+	}
+
+	public void setProviderId(final String providerId) {
+		this.providerId = providerId;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/utils/SimilarityMeshBuilder.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/utils/SimilarityMeshBuilder.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/utils/SimilarityMeshBuilder.java	(revision 43444)
@@ -0,0 +1,33 @@
+package eu.dnetlib.msro.workflows.nodes.utils;
+
+import java.util.List;
+
+import com.google.common.collect.Lists;
+import eu.dnetlib.data.proto.TypeProtos.Type;
+import eu.dnetlib.miscutils.collections.Pair;
+
+public class SimilarityMeshBuilder {
+
+	public static List<Similarity> build(final Type type, final List<String> list) {
+
+		return build(type, list, true);
+	}
+
+	public static List<Similarity> build(final Type type, final List<String> list, final boolean addTypePrefix) {
+		final String typePrefix = String.valueOf(type.getNumber());
+		final List<Similarity> res = Lists.newArrayList();
+		for (int i = 0; i < list.size(); i++) {
+			for (int j = 0; j < list.size(); j++) {
+
+				if (!list.get(i).equals(list.get(j))) {
+
+					final String source = addTypePrefix ? typePrefix + "|" + list.get(i) : list.get(i);
+					final String target = addTypePrefix ? typePrefix + "|" + list.get(j) : list.get(j);
+
+					res.add(new Similarity(new Pair<String, String>(source, target), type));
+				}
+			}
+		}
+		return res;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/openaire/api/objects/PublicationEntry.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/openaire/api/objects/PublicationEntry.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/openaire/api/objects/PublicationEntry.java	(revision 43444)
@@ -0,0 +1,259 @@
+package eu.dnetlib.openaire.api.objects;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.miscutils.functional.hash.Hashing;
+import eu.dnetlib.openaire.api.SinglePublicationSubmitterUtils;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.velocity.app.VelocityEngine;
+import org.apache.velocity.tools.generic.EscapeTool;
+import org.springframework.ui.velocity.VelocityEngineUtils;
+
+/**
+ * Created by michele on 02/12/15.
+ */
+public class PublicationEntry {
+
+	private static final Map<String, Map<String, String>> cached_vocabularies = new HashMap<String, Map<String, String>>();
+	private static final Map<String, DatasourceEntry> cached_datasources = new HashMap<String, DatasourceEntry>();
+	private static final Map<String, String> cached_contexts = new HashMap<String, String>();
+	private static final Log log = LogFactory.getLog(PublicationEntry.class);
+	private static long last_cache_update = 0;
+	private String originalId;
+	private String title;
+	private List<String> authors = new ArrayList<String>();
+	private String publisher;
+	private String description;
+	private String language;
+	private List<PidEntry> pids = new ArrayList<PidEntry>();
+	private String licenseCode;
+	private String embargoEndDate;
+	private String resourceType;
+	private String url;
+	private String collectedFromId;
+	private String hostedById;
+	// String according to openaire guidelines:
+	// info:eu-repo/grantAgreement/Funder/FundingProgram/ProjectID/[Jurisdiction]/[ProjectName]/[ProjectAcronym]
+	private List<String> contexts = new ArrayList<String>();
+	// String according to the EGI context profile, example: egi::classification::natsc::math
+	private List<String> linksToProjects = new ArrayList<String>();
+
+	public PublicationEntry() {
+	}
+
+	private synchronized static DatasourceEntry getDatasourceInfo(final String dsId, final ISLookUpService lookupService) throws ISLookUpException {
+		if (StringUtils
+				.isBlank(dsId)) { return new DatasourceEntry("openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18", "Unknown Repository", "unknown_____"); }
+
+		if (!cached_datasources.containsKey(dsId)) {
+			final String query =
+					"collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')//CONFIGURATION[./DATASOURCE_ORIGINAL_ID='" + dsId
+							+ "']/concat(./OFFICIAL_NAME, ' @@@ ', .//FIELD/value[../key='NamespacePrefix'])";
+			final String s = lookupService.getResourceProfileByQuery(query);
+			final String[] arr = s.split("@@@");
+
+			final DatasourceEntry ds = new DatasourceEntry(dsId, arr[0].trim(), arr[1].trim());
+
+			if (StringUtils.isBlank(ds.getName()) || StringUtils.isBlank(ds.getPrefix())) {
+				log.error("Invalid datasource id: " + dsId);
+				throw new ISLookUpException("Invalid datasource id: " + dsId);
+			} else {
+				cached_datasources.put(dsId, ds);
+			}
+		}
+
+		return cached_datasources.get(dsId);
+
+	}
+
+	private synchronized static Map<String, String> getVocabulary(final String voc, final ISLookUpService lookupService) throws ISLookUpException {
+
+		if (((DateUtils.now() - last_cache_update) < TimeUnit.MINUTES.toMillis(15)) && cached_vocabularies.containsKey(voc)) {
+			return cached_vocabularies.get(voc);
+		} else {
+			final String query = "collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType')[.//VOCABULARY_NAME/@code='" + voc
+					+ "']//TERM/concat(@code, ' @@@ ', @english_name)";
+
+			final Map<String, String> map = new HashMap<String, String>();
+			for (final String s : lookupService.quickSearchProfile(query)) {
+				final String[] arr = s.split("@@@");
+				map.put(arr[0].trim(), arr[1].trim());
+			}
+
+			cached_vocabularies.put(voc, map);
+
+			last_cache_update = DateUtils.now();
+
+			return map;
+		}
+	}
+
+	private synchronized static Map<String, String> getContexts(final ISLookUpService lookupService) throws ISLookUpException {
+		if (((DateUtils.now() - last_cache_update) > TimeUnit.MINUTES.toMillis(15)) || cached_contexts.isEmpty()) {
+			final String query =
+					"collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')[.//context/@type='community']//*[name()='context' or name()='category' or name()='concept']/concat(@id, ' @@@ ', @label)";
+
+			cached_contexts.clear();
+			for (final String s : lookupService.quickSearchProfile(query)) {
+				final String[] arr = s.split("@@@");
+				cached_contexts.put(arr[0].trim(), arr[1].trim());
+			}
+			last_cache_update = DateUtils.now();
+		}
+		return cached_contexts;
+	}
+
+	public String getEmbargoEndDate() {
+		return embargoEndDate;
+	}
+
+	public void setEmbargoEndDate(final String embargoEndDate) {
+		this.embargoEndDate = embargoEndDate;
+	}
+
+	public String getOriginalId() {
+		return originalId;
+	}
+
+	public void setOriginalId(final String originalId) {
+		this.originalId = originalId;
+	}
+
+	public String getTitle() {
+		return title;
+	}
+
+	public void setTitle(final String title) {
+		this.title = title;
+	}
+
+	public List<String> getAuthors() {
+		return authors;
+	}
+
+	public void setAuthors(final List<String> authors) {
+		this.authors = authors;
+	}
+
+	public String getPublisher() {
+		return publisher;
+	}
+
+	public void setPublisher(final String publisher) {
+		this.publisher = publisher;
+	}
+
+	public String getDescription() {
+		return description;
+	}
+
+	public void setDescription(final String description) {
+		this.description = description;
+	}
+
+	public String getLanguage() {
+		return language;
+	}
+
+	public void setLanguage(final String language) {
+		this.language = language;
+	}
+
+	public List<PidEntry> getPids() {
+		return pids;
+	}
+
+	public void setPids(final List<PidEntry> pids) {
+		this.pids = pids;
+	}
+
+	public String getLicenseCode() {
+		return licenseCode;
+	}
+
+	public void setLicenseCode(final String licenseCode) {
+		this.licenseCode = licenseCode;
+	}
+
+	public String getResourceType() {
+		return resourceType;
+	}
+
+	public void setResourceType(final String resourceType) {
+		this.resourceType = resourceType;
+	}
+
+	public String getUrl() {
+		return url;
+	}
+
+	public void setUrl(final String url) {
+		this.url = url;
+	}
+
+	public String getCollectedFromId() {
+		return collectedFromId;
+	}
+
+	public void setCollectedFromId(final String collectedFromId) {
+		this.collectedFromId = collectedFromId;
+	}
+
+	public String getHostedById() {
+		return hostedById;
+	}
+
+	public void setHostedById(final String hostedById) {
+		this.hostedById = hostedById;
+	}
+
+	public List<String> getContexts() {
+		return contexts;
+	}
+
+	public void setContexts(final List<String> contexts) {
+		this.contexts = contexts;
+	}
+
+	public List<String> getLinksToProjects() {
+		return linksToProjects;
+	}
+
+	public void setLinksToProjects(final List<String> linksToProjects) {
+		this.linksToProjects = linksToProjects;
+	}
+
+	public String asOafRecord(final VelocityEngine ve,
+			final ISLookUpService lookupService,
+			final String oafSchemaLocation) throws Exception {
+
+		final DatasourceEntry collectedFromEntry = getDatasourceInfo(collectedFromId, lookupService);
+		final DatasourceEntry hostedByEntry = getDatasourceInfo(hostedById, lookupService);
+
+		final String objId = collectedFromEntry.getPrefix() + "::" + Hashing.md5(originalId);
+
+		final Map<String, Object> model = new HashMap<String, Object>();
+		model.put("esc", new EscapeTool());
+		model.put("util", new SinglePublicationSubmitterUtils(lookupService));
+		model.put("pub", this);
+		model.put("objIdentifier", objId);
+		model.put("oafSchemaLocation", oafSchemaLocation);
+		model.put("licenses", getVocabulary("dnet:access_modes", lookupService));
+		model.put("resourceTypes", getVocabulary("dnet:publication_resource", lookupService));
+		model.put("pidTypes", getVocabulary("dnet:pid_types", lookupService));
+		model.put("languages", getVocabulary("dnet:languages", lookupService));
+		model.put("contexts", getContexts(lookupService));
+		model.put("dateOfCollection", (new SimpleDateFormat("yyyy-MM-dd\'T\'hh:mm:ss\'Z\'")).format(new Date()));
+		model.put("collectedFrom", collectedFromEntry);
+		model.put("hostedBy", hostedByEntry);
+
+		return VelocityEngineUtils.mergeTemplateIntoString(ve, "/eu/dnetlib/msro/openaireplus/api/indexRecord.xml.vm", "UTF-8", model);
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/utils/Entity.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/utils/Entity.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/utils/Entity.java	(revision 43444)
@@ -0,0 +1,91 @@
+package eu.dnetlib.msro.workflows.nodes.dedup.utils;
+
+import com.google.gson.GsonBuilder;
+
+/**
+ * The Class Entity.
+ */
+public class Entity {
+
+	/**
+	 * The name.
+	 */
+	private String name;
+
+	/**
+	 * The code.
+	 */
+	private String code;
+
+	/**
+	 * The label.
+	 */
+	private String label;
+
+	public Entity() {
+	}
+
+	/**
+	 * Instantiates a new entity.
+	 *
+	 * @param name  the name
+	 * @param code  the code
+	 * @param label the label
+	 */
+	public Entity(final String name, final String code, final String label) {
+		super();
+		this.setName(name);
+		this.setCode(code);
+		this.setLabel(label);
+	}
+
+	/**
+	 * Gets the name.
+	 *
+	 * @return the name
+	 */
+	public String getName() {
+		return name;
+	}
+
+	public void setName(final String name) {
+		this.name = name;
+	}
+
+	/**
+	 * Gets the code.
+	 *
+	 * @return the code
+	 */
+	public String getCode() {
+		return code;
+	}
+
+	public void setCode(final String code) {
+		this.code = code;
+	}
+
+	/**
+	 * Gets the label.
+	 *
+	 * @return the label
+	 */
+	public String getLabel() {
+		return label;
+	}
+
+	public void setLabel(final String label) {
+		this.label = label;
+	}
+
+	/*
+	 * (non-Javadoc)
+	 *
+	 * @see java.lang.Object#toString()
+	 */
+	@Override
+	public String toString() {
+		return new GsonBuilder().setPrettyPrinting().create().toJson(this);
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/datacite/SplitDatasetRecord.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/datacite/SplitDatasetRecord.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/datacite/SplitDatasetRecord.java	(revision 43444)
@@ -0,0 +1,107 @@
+package eu.dnetlib.msro.workflows.nodes.datacite;
+
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import eu.dnetlib.enabling.resultset.client.ResultSetClient;
+import eu.dnetlib.enabling.resultset.factory.ResultSetFactory;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.common.ResultSet;
+import org.springframework.beans.factory.annotation.Autowired;
+
+/**
+ * The Class SplitDatasetRecord.
+ */
+public class SplitDatasetRecord extends SimpleJobNode {
+
+	/**
+	 * The executor.
+	 */
+	private final Executor executor = Executors.newSingleThreadExecutor();
+	/**
+	 * The input epr parm.
+	 */
+	private String inputEprParm;
+	/**
+	 * The output epr parm.
+	 */
+	private String outputEprParm;
+	/**
+	 * The result set client factory.
+	 */
+	@Autowired
+	private ResultSetClient resultSetClient;
+	/**
+	 * The result set factory.
+	 */
+	@Autowired
+	private ResultSetFactory resultSetFactory;
+
+	/*
+	 * (non-Javadoc)
+	 *
+	 * @see eu.dnetlib.msro.workflows.nodes.SimpleJobNode#execute(com.googlecode.sarasvati.NodeToken)
+	 */
+	@Override
+	protected String execute(final Env env) throws Exception {
+		final ResultSet<?> inputEpr = env.getAttribute(this.inputEprParm, ResultSet.class);
+		final Iterable<String> input = this.resultSetClient.iter(inputEpr, String.class);
+		final LinkedBlockingQueue<String> publicationsQueue = new LinkedBlockingQueue<String>();
+		final SplitterDatasetsIterator splitterIterator = new SplitterDatasetsIterator(publicationsQueue, input, "publications");
+
+		this.executor.execute(new Runnable() {
+
+			@Override
+			public void run() {
+				splitterIterator.populateQueues();
+			}
+		});
+
+		final ResultSet<String> eprUrls_publications =
+				this.resultSetFactory.createResultSet(() -> new IteratorOnQueue<String>(publicationsQueue, SplitterDatasetsIterator.END_QUEUE));
+
+		env.setAttribute(getOutputEprParm(), eprUrls_publications);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	/**
+	 * Gets the input epr parm.
+	 *
+	 * @return the inputEprParm
+	 */
+	public String getInputEprParm() {
+		return this.inputEprParm;
+	}
+
+	/**
+	 * Sets the input epr parm.
+	 *
+	 * @param inputEprParm the inputEprParm to set
+	 */
+	public void setInputEprParm(final String inputEprParm) {
+		this.inputEprParm = inputEprParm;
+	}
+
+	/**
+	 * Gets the output epr parm.
+	 *
+	 * @return the output epr parm
+	 */
+	public String getOutputEprParm() {
+		return this.outputEprParm;
+	}
+
+	/**
+	 * Sets the output epr parm.
+	 *
+	 * @param outputEprParm the new output epr parm
+	 */
+	public void setOutputEprParm(final String outputEprParm) {
+		this.outputEprParm = outputEprParm;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/openaire/api/SinglePublicationSubmitterUtils.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/openaire/api/SinglePublicationSubmitterUtils.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/openaire/api/SinglePublicationSubmitterUtils.java	(revision 43444)
@@ -0,0 +1,165 @@
+package eu.dnetlib.openaire.api;
+
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import eu.dnetlib.miscutils.functional.hash.Hashing;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * Created by michele on 15/01/16.
+ */
+public class SinglePublicationSubmitterUtils {
+
+	private final ISLookUpService lookupService;
+
+	public SinglePublicationSubmitterUtils(final ISLookUpService lookupService) {
+		this.lookupService = lookupService;
+	}
+
+	public Map<String, String> calculateProjectInfo(final String link) {
+		final Map<String, String> info = new HashMap<String, String>();
+		final String[] arr = link.split("/");
+		//info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble
+
+		final String funderId = calculateFunderId(arr[2]);
+		info.put("id", calculateProjectId(arr[2], arr[3], arr[4]));
+		info.put("funderShortName", arr[2]);
+		info.put("fundingName", arr[3]);
+		info.put("code", arr[4]);
+		info.put("jurisdiction", arr[5]);
+		info.put("title", StringUtils.isNotBlank(arr[6]) ? arr[6] : arr[7]);
+		info.put("acronym", arr[7]);
+		info.put("funderId", funderId);
+		info.put("funderName", calculateFunderName(arr[2]));
+		info.put("fundingId", funderId + "::" + arr[3]);
+
+		return info;
+	}
+
+	private String calculateProjectId(final String funderShortName, String funding, String code) {
+		final String suffix = Hashing.md5(code);
+
+		if (funderShortName.equalsIgnoreCase("ec")) {
+			if (funding.equalsIgnoreCase("fp7")) {
+				return "corda_______::" + suffix;
+			} else {
+				return "corda__h2020::" + suffix;
+			}
+		} else if (funderShortName.equalsIgnoreCase("arc")) {
+			return "arc_________::" + suffix;
+		} else if (funderShortName.equalsIgnoreCase("fct")) {
+			return "fct_________::" + suffix;
+		} else if (funderShortName.equalsIgnoreCase("nhmrc")) {
+			return "nhmrc_______::" + suffix;
+		} else if (funderShortName.equalsIgnoreCase("wt")) {
+			return "wt__________::" + suffix;
+		} else {
+			return "";
+		}
+	}
+
+	private String calculateFunderId(final String funderShortName) {
+		if (funderShortName.equalsIgnoreCase("ec")) {
+			return "ec__________::EC";
+		} else if (funderShortName.equalsIgnoreCase("arc")) {
+			return "arc_________::ARC";
+		} else if (funderShortName.equalsIgnoreCase("fct")) {
+			return "fct_________::FCT";
+		} else if (funderShortName.equalsIgnoreCase("nhmrc")) {
+			return "nhmrc_______::NHMRC";
+		} else if (funderShortName.equalsIgnoreCase("wt")) {
+			return "wt__________::WT";
+		} else {
+			return "";
+		}
+	}
+
+	private String calculateFunderName(final String funderShortName) {
+		if (funderShortName.equalsIgnoreCase("ec")) {
+			return "European Commission";
+		} else if (funderShortName.equalsIgnoreCase("arc")) {
+			return "Australian Research Council (ARC)";
+		} else if (funderShortName.equalsIgnoreCase("fct")) {
+			return "FundaÃ§Ã£o para a CiÃªncia e a Tecnologia";
+		} else if (funderShortName.equalsIgnoreCase("nhmrc")) {
+			return "National Health and Medical Research Council (NHMRC)";
+		} else if (funderShortName.equalsIgnoreCase("wt")) {
+			return "Wellcome Trust";
+		} else {
+			return "";
+		}
+	}
+
+	public List<ContextInfo> processContexts(List<String> list) {
+		return Lists.newArrayList(Lists.transform(list, new Function<String, ContextInfo>() {
+			@Override
+			public ContextInfo apply(final String s) {
+				return createContextInfo(s.split("::"), 0);
+			}
+
+			private ContextInfo createContextInfo(final String[] arr, final int pos) {
+				final StringWriter id = new StringWriter();
+				id.write(arr[0]);
+				for (int i = 0; i < pos; i++) {
+					id.write("::");
+					id.write(arr[i + 1]);
+				}
+				final String elem = (pos == 0) ? "context" : (pos == 1) ? "category" : "concept";
+				final ContextInfo info = new ContextInfo(elem, id.toString());
+				if (pos + 1 < arr.length) {
+					info.getChildren().add(createContextInfo(arr, pos + 1));
+				}
+				return info;
+			}
+		}));
+
+	}
+
+	public class ContextInfo {
+
+		private String elem;
+		private String id;
+		private List<ContextInfo> children = new ArrayList<ContextInfo>();
+
+		public ContextInfo(final String elem,
+				final String id) {
+			this.elem = elem;
+			this.id = id;
+		}
+
+		public String getElem() {
+			return elem;
+		}
+
+		public void setElem(final String elem) {
+			this.elem = elem;
+		}
+
+		public String getId() {
+			return id;
+		}
+
+		public void setId(final String id) {
+			this.id = id;
+		}
+
+		public List<ContextInfo> getChildren() {
+			return children;
+		}
+
+		public void setChildren(final List<ContextInfo> children) {
+			this.children = children;
+		}
+
+		public boolean isRoot() {
+			return elem.equals("context");
+		}
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/openaire/api/RecentPublicationsQueue.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/openaire/api/RecentPublicationsQueue.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/openaire/api/RecentPublicationsQueue.java	(revision 43444)
@@ -0,0 +1,94 @@
+package eu.dnetlib.openaire.api;
+
+import java.io.StringReader;
+import java.util.Iterator;
+import java.util.List;
+
+import com.mongodb.*;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.io.SAXReader;
+import org.springframework.beans.factory.annotation.Required;
+
+/**
+ * Created by michele on 11/11/15.
+ */
+public class RecentPublicationsQueue implements Iterable<String> {
+
+	private static final Log log = LogFactory.getLog(RecentPublicationsQueue.class);
+
+	private DB db;
+	private String collection;
+
+	public void init() {
+		if (!db.collectionExists(collection)) {
+			log.info(String.format("creating collection %s", collection));
+			db.createCollection(collection, new BasicDBObject());
+		}
+	}
+
+	@Override
+	public Iterator<String> iterator() {
+
+		final DBCursor cursor = db.getCollection(collection).find();
+
+		return new Iterator<String>() {
+
+			@Override
+			public boolean hasNext() {
+				return cursor.hasNext();
+			}
+
+			@Override
+			public String next() {
+				final DBObject obj = cursor.next();
+				return ((obj != null) && obj.containsField("record")) ? obj.get("record").toString() : "";
+			}
+
+			@Override
+			public void remove() {
+				throw new RuntimeException("NOT IMPLEMENTED");
+			}
+		};
+	}
+
+	synchronized public void add(final String oaf) throws Exception {
+		final String id = (new SAXReader()).read(new StringReader(oaf)).valueOf("//*[local-name() = 'objIdentifier']");
+
+		log.info("Saving record " + id + " in db: " + db.getName() + ", coll: " + collection);
+
+		final DBCollection coll = db.getCollection(collection);
+		final DBObject obj = BasicDBObjectBuilder.start()
+				.append("id", id)
+				.append("record", oaf)
+				.append("date", DateUtils.now())
+				.get();
+		coll.update(new BasicDBObject("id", id), obj, true, false);
+	}
+
+	public void remove(final List<String> list) {
+		final DBCollection coll = db.getCollection(collection);
+		for (final String id : list) {
+			coll.remove(new BasicDBObject("id", id));
+		}
+	}
+
+	public DB getDb() {
+		return db;
+	}
+
+	@Required
+	public void setDb(final DB db) {
+		this.db = db;
+	}
+
+	public String getCollection() {
+		return collection;
+	}
+
+	@Required
+	public void setCollection(final String collection) {
+		this.collection = collection;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/openaire/api/SinglePublicationSubmitter.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/openaire/api/SinglePublicationSubmitter.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/openaire/api/SinglePublicationSubmitter.java	(revision 43444)
@@ -0,0 +1,122 @@
+package eu.dnetlib.openaire.api;
+
+import javax.annotation.Resource;
+
+import com.google.common.base.Function;
+import com.google.gson.Gson;
+import eu.dnetlib.data.index.CloudIndexClient;
+import eu.dnetlib.data.index.CloudIndexClientFactory;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.index.OafToIndexRecordFactory;
+import eu.dnetlib.openaire.api.objects.PublicationEntry;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.velocity.app.VelocityEngine;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.*;
+
+/**
+ * Created by michele on 11/11/15.
+ */
+@Controller
+public class SinglePublicationSubmitter {
+
+	@Value(value = "oaf.schema.location")
+	private String oafSchemaLocation;
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	@Resource
+	private OafToIndexRecordFactory oafToIndexRecordFactory;
+
+	@Resource
+	private RecentPublicationsQueue recentPublicationsQueue;
+
+	@Resource(name = "openaireplusApisVelocityEngine")
+	private VelocityEngine velocityEngine;
+
+	@Value(value = "${openaireplus.msro.api.findSolrIndexUrl.xquery}")
+	private ClassPathResource findSolrIndexUrl;
+
+	@Value(value = "${openaireplus.msro.api.findIndexDsInfo.xquery}")
+	private ClassPathResource findIndexDsInfo;
+
+	@RequestMapping(value = "/api/publications/feedJson", method = RequestMethod.POST)
+	public
+	@ResponseBody
+	boolean addPublication(@RequestParam(value = "json", required = true) final String json) throws MSROException {
+		final PublicationEntry pub = new Gson().fromJson(json, PublicationEntry.class);
+		return addPublication(pub);
+	}
+
+	@RequestMapping(value = "/api/publications/feedObject", method = RequestMethod.POST)
+	public
+	@ResponseBody
+	boolean addPublication(@RequestBody final PublicationEntry pub) throws MSROException {
+
+		if (StringUtils.isBlank(pub.getOriginalId())) {
+			throw new MSROException("A required field is missing: originalId");
+		}
+		if (StringUtils.isBlank(pub.getTitle())) {
+			throw new MSROException("A required field is missing: title");
+		}
+		if (StringUtils.isBlank(pub.getUrl())) {
+			throw new MSROException("A required field is missing: url");
+		}
+		if (StringUtils.isBlank(pub.getLicenseCode())) {
+			throw new MSROException("A required field is missing: licenceCode");
+		}
+		if (StringUtils.isBlank(pub.getResourceType())) {
+			throw new MSROException("A required field is missing: resourceType");
+		}
+		if (StringUtils.isBlank(pub.getCollectedFromId())) {
+			throw new MSROException("A required field is missing: collectedFromId");
+		}
+
+		/*if (StringUtils.isBlank(pub.getHostedById())) {
+			throw new MSROException("A required field is missing: hostedById");
+		}*/
+
+		CloudIndexClient idxClient = null;
+
+		try {
+			final String baseUrl = calculateIndexBaseUrl();
+			final String[] arr = calculateCurrentIndexDsInfo().split("@@@");
+			final String indexDsId = arr[0].trim();
+			final String format = arr[1].trim();
+			final String coll = arr[2].trim();
+
+			idxClient = CloudIndexClientFactory.newIndexClient(baseUrl, coll, false);
+
+			final String oafRecord = pub.asOafRecord(velocityEngine, serviceLocator.getService(ISLookUpService.class), oafSchemaLocation);
+
+			recentPublicationsQueue.add(oafRecord);
+
+			//TODO check runtime works
+			final Function<String, String> xslt = (Function<String, String>) oafToIndexRecordFactory.newTransformer(format);
+			return (idxClient.feed(oafRecord, indexDsId, xslt) == 0);
+		} catch (Throwable e) {
+			throw new MSROException("Error adding publication: " + e.getMessage(), e);
+		} finally {
+			if (idxClient != null) {
+				idxClient.close();
+			}
+		}
+	}
+
+	private String calculateCurrentIndexDsInfo() throws Exception {
+		final String query = IOUtils.toString(findIndexDsInfo.getInputStream());
+		return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(query);
+	}
+
+	private String calculateIndexBaseUrl() throws Exception {
+		final String query = IOUtils.toString(findSolrIndexUrl.getInputStream());
+		return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(query);
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/DedupConfigurationSetterJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/DedupConfigurationSetterJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/DedupConfigurationSetterJobNode.java	(revision 43444)
@@ -0,0 +1,69 @@
+package eu.dnetlib.msro.workflows.nodes.dedup;
+
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.AsyncJobNode;
+import eu.dnetlib.msro.workflows.nodes.dedup.utils.DedupConfigurationOrchestration;
+import eu.dnetlib.msro.workflows.nodes.dedup.utils.DedupConfigurationOrchestrationLoader;
+import eu.dnetlib.msro.workflows.procs.Env;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class DedupConfigurationSetterJobNode extends AsyncJobNode {
+
+	private String dedupConfigSequence;
+
+	private String dedupConfigSequenceParam;
+
+	private String entityTypeNameParam;
+
+	private String entityTypeIdParam;
+
+	@Autowired
+	private DedupConfigurationOrchestrationLoader dedupOrchestrationLoader;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		if (StringUtils.isBlank(getDedupConfigSequence())) throw new IllegalArgumentException("missing configuration sequence");
+
+		final DedupConfigurationOrchestration dedupOrchestration = dedupOrchestrationLoader.loadByActionSetId(getDedupConfigSequence());
+
+		env.setAttribute(getEntityTypeNameParam(), dedupOrchestration.getEntity().getName());
+		env.setAttribute(getEntityTypeIdParam(), dedupOrchestration.getEntity().getCode());
+		env.setAttribute(getDedupConfigSequenceParam(), dedupOrchestration);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getDedupConfigSequence() {
+		return dedupConfigSequence;
+	}
+
+	public void setDedupConfigSequence(final String dedupConfigSequence) {
+		this.dedupConfigSequence = dedupConfigSequence;
+	}
+
+	public String getDedupConfigSequenceParam() {
+		return dedupConfigSequenceParam;
+	}
+
+	public void setDedupConfigSequenceParam(final String dedupConfigSequenceParam) {
+		this.dedupConfigSequenceParam = dedupConfigSequenceParam;
+	}
+
+	public String getEntityTypeNameParam() {
+		return entityTypeNameParam;
+	}
+
+	public void setEntityTypeNameParam(final String entityTypeNameParam) {
+		this.entityTypeNameParam = entityTypeNameParam;
+	}
+
+	public String getEntityTypeIdParam() {
+		return entityTypeIdParam;
+	}
+
+	public void setEntityTypeIdParam(final String entityTypeIdParam) {
+		this.entityTypeIdParam = entityTypeIdParam;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/DedupCheckEntitySequenceJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/DedupCheckEntitySequenceJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/DedupCheckEntitySequenceJobNode.java	(revision 43444)
@@ -0,0 +1,135 @@
+package eu.dnetlib.msro.workflows.nodes.dedup;
+
+import java.util.Queue;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.nodes.dedup.utils.DedupConfigurationOrchestration;
+import eu.dnetlib.msro.workflows.nodes.dedup.utils.DedupConfigurationOrchestrationLoader;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class DedupCheckEntitySequenceJobNode extends SimpleJobNode {
+
+	public static final String DEDUP_GROUPER_LOOPER = "dedup.grouper.looper";
+	private static final Log log = LogFactory.getLog(DedupCheckEntitySequenceJobNode.class);
+	@Autowired
+	private DedupConfigurationOrchestrationLoader dedupOrchestrationLoader;
+
+	private String dedupConfigSequenceParam;
+
+	private String entitySequence;
+
+	private String entityTypeNameParam;
+
+	private String entityTypeIdParam;
+
+	private String actionSetParam;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		if (StringUtils.isBlank(getEntitySequence())) throw new MSROException("missing entity sequence, e.g. a csv: organization,person,result");
+
+		if (env.hasAttribute(DEDUP_GROUPER_LOOPER)) {
+			log.info("reset env variable: " + DEDUP_GROUPER_LOOPER + " to zero");
+			env.setAttribute(DEDUP_GROUPER_LOOPER, 0);
+		}
+
+		if (!env.hasAttribute("entitySequence")) {
+
+			log.info("parsing config sequence: " + getEntitySequence());
+
+			env.setAttribute("entitySequence", getEntitySequence());
+
+			final Iterable<String> sequence = Splitter.on(",").omitEmptyStrings().split(getEntitySequence());
+			final Queue<DedupConfigurationOrchestration> q =
+					Lists.newLinkedList(Iterables.transform(sequence, entityName -> {
+						try {
+							final DedupConfigurationOrchestration dco = Iterables.getFirst(dedupOrchestrationLoader.loadByEntityName(entityName), null);
+							if (dco == null) throw new RuntimeException("unable to find DedupOrchestration profile for entity type: " + entityName);
+							return dco;
+						} catch (final Throwable e) {
+							throw new RuntimeException("", e);
+						}
+					}));
+
+			log.info("built sequence of dedup orchestration profiles, size: " + q.size());
+			final DedupConfigurationOrchestration dco = q.remove();
+			log.info("closing mesh for entity: " + dco.getEntity().getName());
+			setDedupConfParams(env, dco);
+			env.setAttribute("entitySequenceQueue", q);
+
+			return Arc.DEFAULT_ARC;
+		}
+
+		final Queue<DedupConfigurationOrchestration> q = (Queue<DedupConfigurationOrchestration>) env.getAttribute("entitySequenceQueue");
+
+		if (!q.isEmpty()) {
+			log.info("remaining dedup orchestration profiles: " + q.size());
+			final DedupConfigurationOrchestration dco = q.remove();
+			log.info("closing mesh for entity: " + dco.getEntity().getName());
+
+			setDedupConfParams(env, dco);
+			return Arc.DEFAULT_ARC;
+		}
+
+		log.info("completed closing mesh for entities: " + getEntitySequence());
+		return "done";
+
+	}
+
+	private void setDedupConfParams(final Env env, final DedupConfigurationOrchestration dco) {
+		env.setAttribute(getEntityTypeNameParam(), dco.getEntity().getName());
+		env.setAttribute(getEntityTypeIdParam(), dco.getEntity().getCode());
+		env.setAttribute(getActionSetParam(), dco.getActionSetId());
+		env.setAttribute(getDedupConfigSequenceParam(), dco);
+	}
+
+	public String getEntitySequence() {
+		return entitySequence;
+	}
+
+	public void setEntitySequence(final String entitySequence) {
+		this.entitySequence = entitySequence;
+	}
+
+	public String getDedupConfigSequenceParam() {
+		return dedupConfigSequenceParam;
+	}
+
+	public void setDedupConfigSequenceParam(final String dedupConfigSequenceParam) {
+		this.dedupConfigSequenceParam = dedupConfigSequenceParam;
+	}
+
+	public String getEntityTypeNameParam() {
+		return entityTypeNameParam;
+	}
+
+	public void setEntityTypeNameParam(final String entityTypeNameParam) {
+		this.entityTypeNameParam = entityTypeNameParam;
+	}
+
+	public String getEntityTypeIdParam() {
+		return entityTypeIdParam;
+	}
+
+	public void setEntityTypeIdParam(final String entityTypeIdParam) {
+		this.entityTypeIdParam = entityTypeIdParam;
+	}
+
+	public String getActionSetParam() {
+		return actionSetParam;
+	}
+
+	public void setActionSetParam(final String actionSetParam) {
+		this.actionSetParam = actionSetParam;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/DedupConfigurationAwareJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/DedupConfigurationAwareJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/DedupConfigurationAwareJobNode.java	(revision 43444)
@@ -0,0 +1,46 @@
+package eu.dnetlib.msro.workflows.nodes.dedup;
+
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.dedup.utils.DedupConfigurationOrchestration;
+import eu.dnetlib.msro.workflows.nodes.hadoop.SubmitHadoopJobNode;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.pace.config.DedupConfig;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class DedupConfigurationAwareJobNode extends SubmitHadoopJobNode {
+
+	public static final String DEDUP_CONF = "dedup.conf";
+
+	private static final Log log = LogFactory.getLog(DedupConfigurationAwareJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	private DedupConfigurationOrchestration dedupConfigurationOrchestration;
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		super.prepareJob(job, token);
+
+		final DedupConfigurationOrchestration conf = getDedupConfigurationOrchestration();
+		if (conf == null) {
+			throw new MSROException("Cannot find dedup configurations in workflow env");
+		}
+		final DedupConfig currentConf = conf.getConfigurations().peek();
+
+		log.debug("using dedup configuration: '" + currentConf + "'");
+
+		job.getParameters().put(DEDUP_CONF, currentConf.toString());
+
+		token.getEnv().setAttribute(DEDUP_CONF, currentConf);
+	}
+
+	// //////////
+
+	public DedupConfigurationOrchestration getDedupConfigurationOrchestration() {
+		return dedupConfigurationOrchestration;
+	}
+
+	public void setDedupConfigurationOrchestration(final DedupConfigurationOrchestration dedupConfigurationOrchestration) {
+		this.dedupConfigurationOrchestration = dedupConfigurationOrchestration;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/FindComplianceRepositoriesJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/FindComplianceRepositoriesJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/FindComplianceRepositoriesJobNode.java	(revision 43444)
@@ -0,0 +1,66 @@
+package eu.dnetlib.msro.workflows.nodes;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.resultset.factory.ResultSetFactory;
+import eu.dnetlib.miscutils.functional.xml.ApplyXslt;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.common.ResultSet;
+import eu.dnetlib.rmi.data.DatabaseService;
+import org.antlr.stringtemplate.StringTemplate;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+
+public class FindComplianceRepositoriesJobNode extends SimpleJobNode {
+
+	private static final Log log = LogFactory.getLog(FindComplianceRepositoriesJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+	// The following files are in "dnet-openaireplus-datasource-manager-service" module
+	private final Resource sqlSelectDatasourcesTmpl = new ClassPathResource("/eu/dnetlib/enabling/datasources/getDatasources.sql.st");
+	private final Resource xsltFile = new ClassPathResource("/eu/dnetlib/enabling/datasources/repo_2_is.xslt");
+	private String dbName;
+	private String eprParam = "repoEpr";
+	@Autowired
+	private UniqueServiceLocator serviceLocator;
+	@Autowired
+	private ResultSetFactory resultSetFactory;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+		final StringTemplate st = new StringTemplate(IOUtils.toString(this.sqlSelectDatasourcesTmpl.getInputStream()));
+		// st.setAttribute(
+		// "condition",
+		// "datasourceclass = 'entityregistry' OR (openairecompatibilityclass IS NOT null AND openairecompatibilityclass != 'UNKNOWN' AND
+		// openairecompatibilityclass != 'notCompatible')");
+		final String sql = st.toString();
+
+		log.debug("Executing query: " + sql);
+
+		final ResultSet<String> rs = this.serviceLocator.getService(DatabaseService.class).searchSQL(this.dbName, sql);
+		final ResultSet<String> mapped = this.resultSetFactory.map(rs, String.class, new ApplyXslt(this.xsltFile));
+
+		env.setAttribute(getEprParam(), mapped);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getDbName() {
+		return this.dbName;
+	}
+
+	public void setDbName(final String dbName) {
+		this.dbName = dbName;
+	}
+
+	public String getEprParam() {
+		return this.eprParam;
+	}
+
+	public void setEprParam(final String eprParam) {
+		this.eprParam = eprParam;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/PrepareIndexDataJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/PrepareIndexDataJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/PrepareIndexDataJobNode.java	(revision 43444)
@@ -0,0 +1,186 @@
+package eu.dnetlib.msro.workflows.nodes.index;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringReader;
+import java.io.StringWriter;
+import javax.annotation.Resource;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.miscutils.functional.hash.Hashing;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.enabling.ISLookUpDocumentNotFoundException;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+import org.springframework.core.io.ClassPathResource;
+
+public class PrepareIndexDataJobNode extends SimpleJobNode {
+
+	public static final String SEPARATOR = "_";
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(PrepareIndexDataJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	/**
+	 * Stylesheet which transforms a layout to another stylesheet which converts a input record to a index record.
+	 */
+	private String layoutToRecordStylesheet;
+
+	private String outputRecordsPathParam;
+
+	private String rottenRecordsPathParam;
+
+	private String hbaseTable;
+
+	private String oafSchemaLocation;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		log.info("start preparing job");
+
+		final String xslt = prepareXslt(env("format", env), env("layout", env));
+
+		env.setAttribute("index.xslt", xslt);
+
+		if (!StringUtils.isBlank(getOutputRecordsPathParam())) {
+			env.setAttribute(getOutputRecordsPathParam(), "/tmp" + getFileName(env, "indexrecords"));
+		}
+		if (!StringUtils.isBlank(getRottenRecordsPathParam())) {
+			env.setAttribute(getRottenRecordsPathParam(), "/tmp" + getFileName(env, "rottenrecords"));
+		}
+
+		env.setAttribute("index.solr.url", getIndexSolrUrlZk());
+		env.setAttribute("index.solr.collection", getCollectionName(env));
+
+		env.setAttribute("index.shutdown.wait.time", getIndexSolrShutdownWait());
+		env.setAttribute("index.buffer.flush.threshold", getIndexBufferFlushTreshold());
+		env.setAttribute("index.solr.sim.mode", isFeedingSimulationMode());
+
+		env.setAttribute("index.feed.timestamp", DateUtils.now_ISO8601());
+
+		env.setAttribute("", this.oafSchemaLocation); // TODO : inserire il nome della property
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getIndexSolrUrlZk() throws ISLookUpException {
+		return getServiceConfigValue(
+				"for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='IndexServiceResourceType'] return $x//PROTOCOL[./@name='solr']/@address/string()");
+	}
+
+	public String getIndexSolrShutdownWait() throws ISLookUpException {
+		return queryForServiceProperty("solr:feedingShutdownTolerance");
+	}
+
+	public String getIndexBufferFlushTreshold() throws ISLookUpException {
+		return queryForServiceProperty("solr:feedingBufferFlushThreshold");
+	}
+
+	public String isFeedingSimulationMode() throws ISLookUpException {
+		return queryForServiceProperty("solr:feedingSimulationMode");
+	}
+
+	private String queryForServiceProperty(final String key) throws ISLookUpException {
+		return getServiceConfigValue(
+				"for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='IndexServiceResourceType'] return $x//SERVICE_PROPERTIES/PROPERTY[./@ key='"
+						+ key + "']/@value/string()");
+	}
+
+	private String getServiceConfigValue(final String xquery) throws ISLookUpException {
+		log.debug("quering for service property: " + xquery);
+		final String res = this.serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xquery);
+		if (StringUtils.isBlank(res)) { throw new IllegalStateException("unable to find unique service property, xquery: " + xquery); }
+		return res;
+	}
+
+	private String getFileName(final Env env, final String fileNamePrefix) {
+		return "/" + fileNamePrefix + "_" + getHbaseTable() + "_" + env.getAttribute("format") + ".seq";
+	}
+
+	private String getCollectionName(final Env env) {
+		return env("format", env) + SEPARATOR + env("layout", env) + SEPARATOR + env("interpretation", env);
+	}
+
+	private String env(final String s, final Env env) {
+		return env.getAttribute(s, String.class);
+	}
+
+	protected String prepareXslt(final String format, final String layout) throws ISLookUpException, IOException, TransformerException {
+
+		final TransformerFactory factory = TransformerFactory.newInstance();
+		final Transformer layoutTransformer = factory.newTransformer(new StreamSource(new StringReader(readXslt(getLayoutToRecordStylesheet()))));
+
+		final StreamResult layoutToXsltXslt = new StreamResult(new StringWriter());
+
+		layoutTransformer.setParameter("format", format);
+		layoutTransformer.transform(new StreamSource(new StringReader(getLayoutSource(format, layout))), layoutToXsltXslt);
+
+		return new String(Hashing.encodeBase64(layoutToXsltXslt.getWriter().toString()));
+	}
+
+	private String readXslt(final String s) throws IOException {
+		final ClassPathResource resource = new ClassPathResource(s);
+		final InputStream inputStream = resource.getInputStream();
+		return IOUtils.toString(inputStream);
+	}
+
+	private String getLayoutSource(final String format, final String layout) throws ISLookUpDocumentNotFoundException, ISLookUpException {
+		return this.serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(
+				"collection('')//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'MDFormatDSResourceType' and .//NAME='" + format + "']//LAYOUT[@name='" + layout
+						+ "']");
+	}
+
+	public String getLayoutToRecordStylesheet() {
+		return this.layoutToRecordStylesheet;
+	}
+
+	public void setLayoutToRecordStylesheet(final String layoutToRecordStylesheet) {
+		this.layoutToRecordStylesheet = layoutToRecordStylesheet;
+	}
+
+	public String getHbaseTable() {
+		return this.hbaseTable;
+	}
+
+	@Required
+	public void setHbaseTable(final String hbaseTable) {
+		this.hbaseTable = hbaseTable;
+	}
+
+	public String getOutputRecordsPathParam() {
+		return this.outputRecordsPathParam;
+	}
+
+	public void setOutputRecordsPathParam(final String outputRecordsPathParam) {
+		this.outputRecordsPathParam = outputRecordsPathParam;
+	}
+
+	public String getRottenRecordsPathParam() {
+		return this.rottenRecordsPathParam;
+	}
+
+	public void setRottenRecordsPathParam(final String rottenRecordsPathParam) {
+		this.rottenRecordsPathParam = rottenRecordsPathParam;
+	}
+
+	public void setOafSchemaLocation(final String oafSchemaLocation) {
+		this.oafSchemaLocation = oafSchemaLocation;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/DedupConfigurationLoaderJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/DedupConfigurationLoaderJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/DedupConfigurationLoaderJobNode.java	(revision 43444)
@@ -0,0 +1,44 @@
+package eu.dnetlib.msro.workflows.nodes.dedup;
+
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.dedup.utils.DedupConfigurationOrchestration;
+import eu.dnetlib.msro.workflows.nodes.dedup.utils.DedupConfigurationOrchestrationLoader;
+import eu.dnetlib.msro.workflows.nodes.hadoop.SubmitHadoopJobNode;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.pace.config.DedupConfig;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import static eu.dnetlib.msro.workflows.nodes.dedup.DedupConfigurationAwareJobNode.DEDUP_CONF;
+
+public class DedupConfigurationLoaderJobNode extends SubmitHadoopJobNode {
+
+	private static final Log log = LogFactory.getLog(DedupConfigurationLoaderJobNode.class);
+
+	private String dedupConfigSequence;
+
+	@Autowired
+	private DedupConfigurationOrchestrationLoader dedupOrchestrationLoader;
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		super.prepareJob(job, token);
+
+		final DedupConfigurationOrchestration dedupOrchestration = dedupOrchestrationLoader.loadByActionSetId(getDedupConfigSequence());
+		final DedupConfig currentConf = dedupOrchestration.getConfigurations().peek();
+
+		log.debug("using dedup configuration: '" + currentConf + "'");
+
+		job.getParameters().put(DEDUP_CONF, currentConf.toString());
+	}
+
+	public String getDedupConfigSequence() {
+		return dedupConfigSequence;
+	}
+
+	public void setDedupConfigSequence(final String dedupConfigSequence) {
+		this.dedupConfigSequence = dedupConfigSequence;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/SelectWfPathByDedupEntityJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/SelectWfPathByDedupEntityJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/SelectWfPathByDedupEntityJobNode.java	(revision 43444)
@@ -0,0 +1,46 @@
+package eu.dnetlib.msro.workflows.nodes.dedup;
+
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.nodes.dedup.utils.DedupConfigurationOrchestration;
+import eu.dnetlib.msro.workflows.nodes.dedup.utils.DedupConfigurationOrchestrationLoader;
+import eu.dnetlib.msro.workflows.procs.Env;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+/**
+ * Created by claudio on 13/07/16.
+ */
+public class SelectWfPathByDedupEntityJobNode extends SimpleJobNode {
+
+	private static final Log log = LogFactory.getLog(SelectWfPathByDedupEntityJobNode.class);
+
+	@Autowired
+	private DedupConfigurationOrchestrationLoader dedupOrchestrationLoader;
+
+	private String dedupConfigSequence;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		if (StringUtils.isBlank(getDedupConfigSequence())) throw new IllegalArgumentException("missing configuration sequence");
+
+		final DedupConfigurationOrchestration dedupOrchestration = dedupOrchestrationLoader.loadByActionSetId(getDedupConfigSequence());
+
+		final String entityName = dedupOrchestration.getEntity().getName();
+
+		log.info("following workflow path: " + entityName);
+
+		return entityName;
+	}
+
+	public String getDedupConfigSequence() {
+		return dedupConfigSequence;
+	}
+
+	public void setDedupConfigSequence(final String dedupConfigSequence) {
+		this.dedupConfigSequence = dedupConfigSequence;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/PrepareDedupIndexJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/PrepareDedupIndexJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/dedup/PrepareDedupIndexJobNode.java	(revision 43444)
@@ -0,0 +1,148 @@
+package eu.dnetlib.msro.workflows.nodes.dedup;
+
+import javax.annotation.Resource;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class PrepareDedupIndexJobNode extends SimpleJobNode {
+
+	public static final String SEPARATOR = "_";
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(PrepareDedupIndexJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+	private String mdFormat;
+
+	private String layout;
+
+	private String interpretation;
+
+	private String rottenRecordsPathParam;
+
+	private String hbaseTable;
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		log.info("start preparing job");
+
+		final String fields = getFields(getMdFormat(), getLayout());
+		env.setAttribute("index.fields", fields);
+
+		if (StringUtils.isBlank(getRottenRecordsPathParam())) {
+			throw new MSROException("missing rotten records path param");
+		}
+		env.setAttribute(getRottenRecordsPathParam(), "/tmp" + getFileName("rottenrecords"));
+
+		env.setAttribute("index.solr.url", getIndexSolrUrlZk());
+		env.setAttribute("index.solr.collection", getCollectionName());
+
+		env.setAttribute("index.shutdown.wait.time", getIndexSolrShutdownWait());
+		env.setAttribute("index.buffer.flush.threshold", getIndexBufferFlushTreshold());
+		env.setAttribute("index.solr.sim.mode", isFeedingSimulationMode());
+
+		env.setAttribute("index.feed.timestamp", DateUtils.now_ISO8601());
+
+		env.setAttribute("entityType", env.getAttribute("entityType"));
+		env.setAttribute("entityTypeId", env.getAttribute("entityTypeId"));
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	private String getFields(final String format, final String layout) throws ISLookUpException {
+		return isLookup(String
+				.format("<FIELDS>{for $x in collection('')//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'MDFormatDSResourceType' and .//NAME='%s']//LAYOUT[@name='%s']/FIELDS/FIELD return $x[string(@path)]}</FIELDS>",
+						format, layout));
+	}
+
+	public String getIndexSolrUrlZk() throws ISLookUpException {
+		return isLookup(
+				"for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='IndexServiceResourceType'] return $x//PROTOCOL[./@name='solr']/@address/string()");
+	}
+
+	public String getIndexSolrShutdownWait() throws ISLookUpException {
+		return queryForServiceProperty("solr:feedingShutdownTolerance");
+	}
+
+	public String getIndexBufferFlushTreshold() throws ISLookUpException {
+		return queryForServiceProperty("solr:feedingBufferFlushThreshold");
+	}
+
+	public String isFeedingSimulationMode() throws ISLookUpException {
+		return queryForServiceProperty("solr:feedingSimulationMode");
+	}
+
+	private String queryForServiceProperty(final String key) throws ISLookUpException {
+		return isLookup("for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='IndexServiceResourceType'] return $x//SERVICE_PROPERTIES/PROPERTY[./@ key='"
+				+ key + "']/@value/string()");
+	}
+
+	private String isLookup(final String xquery) throws ISLookUpException {
+		log.debug("quering for service property: " + xquery);
+		final String res = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xquery);
+		if (StringUtils.isBlank(res)) throw new IllegalStateException("unable to find unique service property, xquery: " + xquery);
+		return res;
+	}
+
+	private String getFileName(final String fileNamePrefix) {
+		return "/" + fileNamePrefix + "_" + getHbaseTable() + "_" + getMdFormat() + ".seq";
+	}
+
+	private String getCollectionName() {
+		return getMdFormat() + SEPARATOR + getLayout() + SEPARATOR + getInterpretation();
+	}
+
+	public String getHbaseTable() {
+		return hbaseTable;
+	}
+
+	public void setHbaseTable(final String hbaseTable) {
+		this.hbaseTable = hbaseTable;
+	}
+
+	public String getRottenRecordsPathParam() {
+		return rottenRecordsPathParam;
+	}
+
+	public void setRottenRecordsPathParam(final String rottenRecordsPathParam) {
+		this.rottenRecordsPathParam = rottenRecordsPathParam;
+	}
+
+	public String getMdFormat() {
+		return mdFormat;
+	}
+
+	public void setMdFormat(final String mdFormat) {
+		this.mdFormat = mdFormat;
+	}
+
+	public String getLayout() {
+		return layout;
+	}
+
+	public void setLayout(final String layout) {
+		this.layout = layout;
+	}
+
+	public String getInterpretation() {
+		return interpretation;
+	}
+
+	public void setInterpretation(final String interpretation) {
+		this.interpretation = interpretation;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/RunMDStorePluginJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/RunMDStorePluginJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/RunMDStorePluginJobNode.java	(revision 43444)
@@ -0,0 +1,55 @@
+package eu.dnetlib.msro.workflows.nodes;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.rmi.data.MDStoreService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+/**
+ * Created by claudio on 21/03/16.
+ */
+public class RunMDStorePluginJobNode extends BlackboardJobNode {
+
+	private static final Log log = LogFactory.getLog(RunMDStorePluginJobNode.class);
+
+	@Autowired
+	private UniqueServiceLocator serviceLocator;
+
+	private String mdId;
+
+	private String pluginName;
+
+	@Override
+	protected String obtainServiceId(final Env env) {
+		return this.serviceLocator.getServiceId(MDStoreService.class);
+	}
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		job.setAction("RUN_PLUGIN");
+		job.getParameters().put("plugin.name", getPluginName());
+		job.getParameters().put("mdId", getMdId());
+
+		log.debug(job.getParameters());
+	}
+
+	public String getMdId() {
+		return this.mdId;
+	}
+
+	public void setMdId(final String mdId) {
+		this.mdId = mdId;
+	}
+
+	public String getPluginName() {
+		return this.pluginName;
+	}
+
+	public void setPluginName(final String pluginName) {
+		this.pluginName = pluginName;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/EntityGrouperConfigurationLoader.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/EntityGrouperConfigurationLoader.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/EntityGrouperConfigurationLoader.java	(revision 43444)
@@ -0,0 +1,46 @@
+package eu.dnetlib.msro.workflows.nodes.index;
+
+import java.util.List;
+import javax.annotation.Resource;
+
+import com.google.common.collect.Iterables;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+
+public class EntityGrouperConfigurationLoader {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(EntityGrouperConfigurationLoader.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	private String xquery;
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	public String load() throws ISLookUpException {
+
+		log.info("loading EntityGrouperConfigurationDSResourceType: " + getXquery());
+
+		final List<String> conf = this.serviceLocator.getService(ISLookUpService.class).quickSearchProfile(getXquery());
+		if (conf == null || conf.isEmpty()) { throw new IllegalStateException("unable to read entity grouper configuration profile"); }
+		if (conf.size() > 1) { throw new IllegalStateException("found more than one EntityGrouperConfigurationDSResourceType"); }
+
+		return Iterables.getOnlyElement(conf).replaceAll("\\s+", " ").trim();
+	}
+
+	public String getXquery() {
+		return this.xquery;
+	}
+
+	@Required
+	public void setXquery(final String xquery) {
+		this.xquery = xquery;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/FindSearchServiceJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/FindSearchServiceJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/FindSearchServiceJobNode.java	(revision 43444)
@@ -0,0 +1,71 @@
+package eu.dnetlib.msro.workflows.nodes.index;
+
+import java.util.List;
+import javax.annotation.Resource;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Value;
+
+public class FindSearchServiceJobNode extends SimpleJobNode {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(FindSearchServiceJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	@Value(value = "${dnet.openaire.service.search.lookup.xquery}")
+	private String xquery;
+
+	private String xqueryForServiceIdParam;
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @throws ISLookUpException
+	 * @throws MSROException
+	 * @see com.googlecode.sarasvati.mem.MemNode#execute(com.googlecode.sarasvati.Engine, com.googlecode.sarasvati.NodeToken)
+	 */
+	@Override
+	public String execute(final Env env) throws ISLookUpException, MSROException {
+
+		log.info("lookup for search service: " + getXquery());
+
+		final List<String> searchServiceIds = this.serviceLocator.getService(ISLookUpService.class).quickSearchProfile(getXquery());
+
+		if (searchServiceIds.size() > 1) { throw new MSROException("too many search services: " + searchServiceIds); }
+
+		if (searchServiceIds.isEmpty()) {
+			return "notFound";
+		} else {
+			env.setAttribute(getXqueryForServiceIdParam(), getXquery());
+			return "found";
+		}
+	}
+
+	public String getXquery() {
+		return this.xquery;
+	}
+
+	public void setXquery(final String xquery) {
+		this.xquery = xquery;
+	}
+
+	public String getXqueryForServiceIdParam() {
+		return this.xqueryForServiceIdParam;
+	}
+
+	public void setXqueryForServiceIdParam(final String xqueryForServiceIdParam) {
+		this.xqueryForServiceIdParam = xqueryForServiceIdParam;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/contexts/ContextDesc.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/contexts/ContextDesc.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/contexts/ContextDesc.java	(revision 43444)
@@ -0,0 +1,85 @@
+package eu.dnetlib.msro.workflows.nodes.contexts;
+
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+import org.dom4j.DocumentHelper;
+import org.dom4j.Element;
+
+public class ContextDesc {
+
+	private String id;
+	private String label;
+	private String type;
+	private String xml;
+	private Map<String, ContextPart> categories = Maps.newLinkedHashMap();
+	private Map<String, String> dbEntries = Maps.newLinkedHashMap();
+
+	public ContextDesc(final String id, final String label, final String type) {
+		this.id = id;
+		this.label = label;
+		this.type = type;
+	}
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getLabel() {
+		return label;
+	}
+
+	public void setLabel(final String label) {
+		this.label = label;
+	}
+
+	public String getType() {
+		return type;
+	}
+
+	public void setType(final String type) {
+		this.type = type;
+	}
+
+	public String getXml() {
+		return xml;
+	}
+
+	public void setXml(final String xml) {
+		this.xml = xml;
+	}
+
+	public Map<String, ContextPart> getCategories() {
+		return categories;
+	}
+
+	public void setCategories(final Map<String, ContextPart> categories) {
+		this.categories = categories;
+	}
+
+	public Map<String, String> getDbEntries() {
+		return dbEntries;
+	}
+
+	public void setDbEntries(final Map<String, String> dbEntries) {
+		this.dbEntries = dbEntries;
+	}
+
+	public Element asDomElement() {
+		final Element ctxElem = DocumentHelper.createElement("context");
+		ctxElem.addAttribute("id", getId());
+		ctxElem.addAttribute("label", getLabel());
+		ctxElem.addAttribute("type", getType());
+
+		for (ContextPart part : getCategories().values()) {
+			ctxElem.add(part.asDomElement("category"));
+		}
+
+		return ctxElem;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/contexts/ProcessContextsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/contexts/ProcessContextsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/contexts/ProcessContextsJobNode.java	(revision 43444)
@@ -0,0 +1,82 @@
+package eu.dnetlib.msro.workflows.nodes.contexts;
+
+import eu.dnetlib.enabling.resultset.client.ResultSetClient;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider;
+import eu.dnetlib.rmi.common.ResultSet;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class ProcessContextsJobNode extends SimpleJobNode {
+
+	private String eprParam;
+	private String contextObj;
+	private String contextId;
+	private String contextLabel;
+	private String contextType;
+
+	@Autowired
+	private ResultSetClient resultSetClient;
+
+	@Override
+	protected void beforeStart(final Token token) {
+		final ResultSet<?> epr = token.getEnv().getAttribute(getEprParam(), ResultSet.class);
+		token.setProgressProvider(new ResultsetProgressProvider(epr, this.resultSetClient));
+	}
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		final ResultSet<?> epr = env.getAttribute(this.eprParam, ResultSet.class);
+
+		final Iterable<String> iter = this.resultSetClient.iter(epr, String.class);
+
+		final ContextDesc context = ContextUtils.getContext(iter, getContextId(), getContextLabel(), getContextType());
+
+		env.setAttribute(this.contextObj, context);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getEprParam() {
+		return this.eprParam;
+	}
+
+	public void setEprParam(final String eprParam) {
+		this.eprParam = eprParam;
+	}
+
+	public String getContextObj() {
+		return this.contextObj;
+	}
+
+	public void setContextObj(final String contextObj) {
+		this.contextObj = contextObj;
+	}
+
+	public String getContextId() {
+		return this.contextId;
+	}
+
+	public void setContextId(final String contextId) {
+		this.contextId = contextId;
+	}
+
+	public String getContextLabel() {
+		return this.contextLabel;
+	}
+
+	public void setContextLabel(final String contextLabel) {
+		this.contextLabel = contextLabel;
+	}
+
+	public String getContextType() {
+		return this.contextType;
+	}
+
+	public void setContextType(final String contextType) {
+		this.contextType = contextType;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/CleanActionSetsProfileJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/CleanActionSetsProfileJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/CleanActionSetsProfileJobNode.java	(revision 43444)
@@ -0,0 +1,42 @@
+package eu.dnetlib.msro.workflows.nodes.actionmanager;
+
+import javax.annotation.Resource;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.enabling.ISRegistryService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+
+public class CleanActionSetsProfileJobNode extends SimpleJobNode {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(CleanActionSetsProfileJobNode.class);
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	private String xupdate;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+		log.info("updating Action Sets profiles: " + getXupdate());
+		this.serviceLocator.getService(ISRegistryService.class).executeXUpdate(getXupdate());
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getXupdate() {
+		return this.xupdate;
+	}
+
+	@Required
+	public void setXupdate(final String xupdate) {
+		this.xupdate = xupdate;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/SwitchIndexJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/SwitchIndexJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/index/SwitchIndexJobNode.java	(revision 43444)
@@ -0,0 +1,82 @@
+package eu.dnetlib.msro.workflows.nodes.index;
+
+import java.util.List;
+
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * SwitchIndexJobNode performs index switch.
+ *
+ * @author claudio
+ */
+public class SwitchIndexJobNode extends BlackboardJobNode {
+
+	private static final String BB_ACTION_SWITCH_INDEX = "UpdateIndex";
+
+	private String inputIndexIdParam;
+
+	private String outputIndexIdParam;
+
+	private String xqueryForServiceIdParam;
+
+	@Override
+	protected String obtainServiceId(final Env env) {
+		final String xquery = env.getAttribute(getXqueryForServiceIdParam(), String.class);
+		List<String> searchServiceIds;
+		try {
+			searchServiceIds = getServiceLocator().getService(ISLookUpService.class).quickSearchProfile(xquery);
+			if (searchServiceIds.size() > 1) { throw new RuntimeException("Too many SearchService ids found using query: " + xquery); }
+			if (searchServiceIds.size() < 1) { throw new RuntimeException("SearchService id not found using query: " + xquery); }
+			return searchServiceIds.get(0);
+		} catch (final ISLookUpException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		job.setAction(BB_ACTION_SWITCH_INDEX);
+		final String indexId = token.getEnv().getAttribute(getInputIndexIdParam(), String.class);
+
+		checkParam(getInputIndexIdParam(), "output indexId param is missing");
+		checkParam(indexId, "indexId is required to perform switch");
+
+		job.getParameters().put(getOutputIndexIdParam(), indexId);
+	}
+
+	private void checkParam(final String param, final String msg) throws MSROException {
+		if (StringUtils.isBlank(param)) { throw new MSROException(msg); }
+	}
+
+	public String getInputIndexIdParam() {
+		return this.inputIndexIdParam;
+	}
+
+	public void setInputIndexIdParam(final String inputIndexIdParam) {
+		this.inputIndexIdParam = inputIndexIdParam;
+	}
+
+	public String getOutputIndexIdParam() {
+		return this.outputIndexIdParam;
+	}
+
+	public void setOutputIndexIdParam(final String outputIndexIdParam) {
+		this.outputIndexIdParam = outputIndexIdParam;
+	}
+
+	public String getXqueryForServiceIdParam() {
+		return this.xqueryForServiceIdParam;
+	}
+
+	public void setXqueryForServiceIdParam(final String xqueryForServiceIdParam) {
+		this.xqueryForServiceIdParam = xqueryForServiceIdParam;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/contexts/PrepareContextDbEntriesJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/contexts/PrepareContextDbEntriesJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/contexts/PrepareContextDbEntriesJobNode.java	(revision 43444)
@@ -0,0 +1,50 @@
+package eu.dnetlib.msro.workflows.nodes.contexts;
+
+import com.google.common.collect.Iterables;
+import eu.dnetlib.enabling.resultset.factory.ResultSetFactory;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.common.ResultSet;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class PrepareContextDbEntriesJobNode extends SimpleJobNode {
+
+	private static final Log log = LogFactory.getLog(PrepareContextDbEntriesJobNode.class);
+	private String contextObj;
+	private String eprParam;
+	@Autowired
+	private ResultSetFactory resultSetFactory;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		final ContextDesc desc = env.getAttribute(this.contextObj, ContextDesc.class);
+
+		final Iterable<String> iter = Iterables.transform(desc.getDbEntries().entrySet(), ContextUtils.getContextRowTransformer());
+		final ResultSet<String> epr = this.resultSetFactory.createResultSet(iter);
+
+		env.setAttribute(this.eprParam, epr.toString());
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getContextObj() {
+		return this.contextObj;
+	}
+
+	public void setContextObj(final String contextObj) {
+		this.contextObj = contextObj;
+	}
+
+	public String getEprParam() {
+		return this.eprParam;
+	}
+
+	public void setEprParam(final String eprParam) {
+		this.eprParam = eprParam;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/contexts/ContextPart.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/contexts/ContextPart.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/contexts/ContextPart.java	(revision 43444)
@@ -0,0 +1,83 @@
+package eu.dnetlib.msro.workflows.nodes.contexts;
+
+import java.util.Map;
+import java.util.Map.Entry;
+
+import com.google.common.collect.Maps;
+import org.dom4j.DocumentHelper;
+import org.dom4j.Element;
+
+public class ContextPart {
+
+	private String id;
+	private String label;
+	private Map<String, String> params = Maps.newLinkedHashMap();
+	private Map<String, ContextPart> parts = Maps.newLinkedHashMap();
+
+	public ContextPart() {
+	}
+
+	public ContextPart(final String id, final String label) {
+		this.id = id;
+		this.label = label;
+	}
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getLabel() {
+		return label;
+	}
+
+	public void setLabel(final String label) {
+		this.label = label;
+	}
+
+	public Map<String, String> getParams() {
+		return params;
+	}
+
+	public void setParams(final Map<String, String> params) {
+		this.params = params;
+	}
+
+	public Map<String, ContextPart> getParts() {
+		return parts;
+	}
+
+	public void setParts(final Map<String, ContextPart> parts) {
+		this.parts = parts;
+	}
+
+	public void addPart(final ContextPart part) {
+		if (parts.containsKey(part.getId())) {
+			final ContextPart localChild = getParts().get(part.getId());
+			for (ContextPart child : part.getParts().values()) {
+				localChild.addPart(child);
+			}
+		} else {
+			parts.put(part.getId(), part);
+		}
+	}
+
+	public Element asDomElement(final String name) {
+		final Element elem = DocumentHelper.createElement(name);
+		elem.addAttribute("id", id);
+		elem.addAttribute("label", label);
+		elem.addAttribute("claim", "false");
+		for (Entry<String, String> e : params.entrySet()) {
+			final Element p = elem.addElement("param");
+			p.addAttribute("name", e.getKey());
+			p.setText(e.getValue());
+		}
+		for (ContextPart child : parts.values()) {
+			elem.add(child.asDomElement("concept"));
+		}
+		return elem;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/UpdateSetsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/UpdateSetsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/UpdateSetsJobNode.java	(revision 43444)
@@ -0,0 +1,86 @@
+package eu.dnetlib.msro.workflows.nodes.actionmanager;
+
+import java.io.StringReader;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Resource;
+
+import com.google.gson.Gson;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import eu.dnetlib.rmi.enabling.ISRegistryService;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Attribute;
+import org.dom4j.Document;
+import org.dom4j.Element;
+import org.dom4j.io.SAXReader;
+
+public class UpdateSetsJobNode extends SimpleJobNode {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(UpdateSetsJobNode.class);
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		@SuppressWarnings("unchecked")
+		final List<Map<String, String>> sets = new Gson().fromJson(env.getAttribute("sets", String.class), List.class);
+
+		final String lastUpdate = DateUtils.now_ISO8601();
+		for (final Map<String, String> set : sets) {
+
+			// update only the enabled sets.
+			if (isEnabled(set)) {
+				log.info("updating set: " + set.toString());
+				addLatestRawSet(set, lastUpdate);
+			} else {
+				log.info("skip set update: " + set.toString());
+			}
+		}
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	private boolean isEnabled(final Map<String, String> set) {
+		return set.containsKey("enabled") && set.get("enabled").equals("true");
+	}
+
+	public void addLatestRawSet(final Map<String, String> set, final String lastUpdate) throws MSROException {
+		final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') where $x//SET/@id = '" + set.get("set")
+				+ "' return $x";
+		try {
+			final String profile = this.serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(q);
+			final Document doc = new SAXReader().read(new StringReader(profile));
+			final String profId = doc.valueOf("//RESOURCE_IDENTIFIER/@value");
+			final Element latest = (Element) doc.selectSingleNode("//RAW_SETS/LATEST");
+			final Element expired = ((Element) doc.selectSingleNode("//RAW_SETS")).addElement("EXPIRED");
+
+			for (final Object o : latest.attributes()) {
+				final Attribute a = (Attribute) o;
+				expired.addAttribute(a.getName(), a.getValue());
+			}
+
+			latest.addAttribute("id", set.get("rawset"));
+			latest.addAttribute("creationDate", set.get("creationDate"));
+			latest.addAttribute("lastUpdate", lastUpdate);
+
+			this.serviceLocator.getService(ISRegistryService.class).updateProfile(profId, doc.asXML(), "ActionManagerSetDSResourceType");
+		} catch (final Exception e) {
+			final String msg = "Error updating profile of set: " + set;
+			log.error(msg);
+			throw new MSROException(msg, e);
+		}
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/PromoteActionsHDFSJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/PromoteActionsHDFSJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/PromoteActionsHDFSJobNode.java	(revision 43444)
@@ -0,0 +1,52 @@
+package eu.dnetlib.msro.workflows.nodes.actionmanager;
+
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.rmi.data.hadoop.actionmanager.ActionManagerService;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * Created by claudio on 14/04/16.
+ */
+public class PromoteActionsHDFSJobNode extends BlackboardJobNode {
+
+	public static final String ALL_SETS = "ALL SETS";
+
+	private String set;
+
+	private String tableName;
+
+	@Override
+	protected String obtainServiceId(final Env env) {
+		return getServiceLocator().getServiceId(ActionManagerService.class);
+	}
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		job.setAction("PROMOTE_FROM_HDFS");
+		if (!StringUtils.isBlank(getSet()) && !getSet().equals(ALL_SETS)) {
+			job.getParameters().put("set", getSet());
+			token.getEnv().setAttribute("set", getSet());
+		}
+
+		job.getParameters().put("targetTable", getTableName());
+	}
+
+	public String getSet() {
+		return this.set;
+	}
+
+	public void setSet(final String set) {
+		this.set = set;
+	}
+
+	public String getTableName() {
+		return tableName;
+	}
+
+	public void setTableName(final String tableName) {
+		this.tableName = tableName;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/PrepareActionSetsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/PrepareActionSetsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/actionmanager/PrepareActionSetsJobNode.java	(revision 43444)
@@ -0,0 +1,92 @@
+package eu.dnetlib.msro.workflows.nodes.actionmanager;
+
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Maps;
+import com.google.gson.Gson;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.data.hadoop.actionmanager.RawSet;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class PrepareActionSetsJobNode extends SimpleJobNode {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(PrepareActionSetsJobNode.class);
+
+	@Autowired
+	private UniqueServiceLocator serviceLocator;
+
+	private String sets;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		final List<Map<String, String>> setList = getSetList();
+		final String now = DateUtils.now_ISO8601();
+
+		final String basePath = this.serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(
+				"/RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value='ActionManagerServiceResourceType']//SERVICE_PROPERTIES/PROPERTY[@key='basePath']/@value/string()");
+
+		for (final Map<String, String> set : setList) {
+
+			set.put("rawset", RawSet.newInstance().getId());
+			set.put("creationDate", now);
+			set.put("path", getPath(basePath, set));
+
+			if (set.get("enabled").equals("true")) {
+				log.info("preparing set: " + simplifySetInfo(set));
+			}
+			// setting the job properties needed to name the rawsets
+			env.setAttribute(set.get("jobProperty"), set.get("rawset"));
+		}
+
+		env.setAttribute("sets", new Gson().toJson(setList));
+		env.setAttribute("actionManagerBasePath", basePath);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	private String getPath(final String basePath, final Map<String, String> set) {
+		final StringBuilder sb = new StringBuilder();
+		sb.append(basePath).append("/").append(set.get("set")).append("/").append(set.get("rawset"));
+		final String path = sb.toString();
+		log.debug("path: " + path);
+		return path;
+	}
+
+	private Map<String, String> simplifySetInfo(final Map<String, String> set) {
+		return Maps.filterKeys(set, new Predicate<String>() {
+
+			@Override
+			public boolean apply(final String k) {
+
+				return k.equals("set") || k.equals("rawset");
+			}
+		});
+	}
+
+	@SuppressWarnings("unchecked")
+	protected List<Map<String, String>> getSetList() {
+		return new Gson().fromJson(getSets(), List.class);
+	}
+
+	public String getSets() {
+		return this.sets;
+	}
+
+	public void setSets(final String sets) {
+		this.sets = sets;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/SubmitHadoopJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/SubmitHadoopJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/SubmitHadoopJobNode.java	(revision 43444)
@@ -0,0 +1,136 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import javax.annotation.Resource;
+
+import com.google.common.collect.Iterables;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.rmi.data.hadoop.HadoopBlackboardActions;
+import eu.dnetlib.rmi.data.hadoop.HadoopJobType;
+import eu.dnetlib.rmi.data.hadoop.HadoopService;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class SubmitHadoopJobNode extends BlackboardJobNode {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(SubmitHadoopJobNode.class);
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	private String hadoopJob;
+
+	private String cluster;
+
+	private boolean simulation = false;
+
+	private Map<String, Object> jobParams = new HashMap<>();
+
+	@Override
+	protected String obtainServiceId(final Env env) {
+		return getServiceLocator().getServiceId(HadoopService.class);
+	}
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		String type = getJobType(getHadoopJob());
+
+		log.info("submitting job " + getHadoopJob() + " type: " + type);
+
+		job.setAction(type);
+		job.getParameters().put("job.name", getHadoopJob());
+		job.getParameters().put("cluster", cluster(token));
+		job.getParameters().put("simulation", String.valueOf(isSimulation()));
+
+		for (Entry<String, Object> e : getJobParams().entrySet()) {
+			try {
+				job.getParameters().put(e.getKey(), e.getValue().toString());
+			} catch (Throwable ex) {
+				throw new MSROException(String.format("missing value for map key '%s'", e.getKey()));
+			}
+		}
+	}
+
+	private String cluster(final Token token) {
+		if (token.getEnv().hasAttribute("cluster")) {
+			String cluster = token.getEnv().getAttribute("cluster", String.class);
+			log.info("found override value in wfEnv for 'cluster' param: " + cluster);
+			return cluster;
+		}
+		return getCluster();
+	}
+
+	/**
+	 * reads the job type for the given job name
+	 *
+	 * @param jobName
+	 * @return
+	 * @throws ISLookUpException
+	 */
+	private String getJobType(final String jobName) throws ISLookUpException {
+		List<String> res =
+				serviceLocator.getService(ISLookUpService.class).quickSearchProfile(
+						"/RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'HadoopJobConfigurationDSResourceType']//HADOOP_JOB[./@name='" + jobName
+								+ "']/@type/string()");
+		if (res.isEmpty()) { throw new IllegalStateException("unable to find job type for job: " + jobName); }
+
+		final HadoopJobType type = HadoopJobType.valueOf(Iterables.getOnlyElement(res));
+
+		switch (type) {
+		case mapreduce:
+			return HadoopBlackboardActions.SUBMIT_MAPREDUCE_JOB.toString();
+		case admin:
+			return HadoopBlackboardActions.SUBMIT_ADMIN_JOB.toString();
+		case oozie:
+			return HadoopBlackboardActions.SUBMIT_OOZIE_JOB.toString();
+		default:
+			throw new IllegalStateException("undefined job type: " + type.toString());
+		}
+	}
+
+	public String getHadoopJob() {
+		return hadoopJob;
+	}
+
+	public void setHadoopJob(final String hadoopJob) {
+		this.hadoopJob = hadoopJob;
+	}
+
+	public String getCluster() {
+		return cluster;
+	}
+
+	public void setCluster(final String cluster) {
+		this.cluster = cluster;
+	}
+
+	public boolean isSimulation() {
+		return simulation;
+	}
+
+	public void setSimulation(final boolean simulation) {
+		this.simulation = simulation;
+	}
+
+	public Map<String, Object> getJobParams() {
+		return jobParams;
+	}
+
+	public void setJobParams(final Map<String, Object> jobParams) {
+		this.jobParams = jobParams;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/PrepareCopyTableJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/PrepareCopyTableJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/PrepareCopyTableJobNode.java	(revision 43444)
@@ -0,0 +1,174 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import java.util.Map;
+import java.util.Set;
+import javax.annotation.Resource;
+
+import com.google.common.collect.Sets;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.data.hadoop.HadoopService;
+import eu.dnetlib.rmi.data.hadoop.HadoopServiceException;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * The PrepareCopyTableJobNode prepares the parameters needed to run the CopyTable job.
+ */
+public class PrepareCopyTableJobNode extends SimpleJobNode {
+
+	/**
+	 * The Constant ZOOKEEPER_ZNODE_PARENT.
+	 */
+	private static final String ZOOKEEPER_ZNODE_PARENT = "zookeeper.znode.parent";
+
+	/**
+	 * The Constant HBASE_ZOOKEEPER_CLIENT_PORT.
+	 */
+	private static final String HBASE_ZOOKEEPER_CLIENT_PORT = "hbase.zookeeper.client.port";
+
+	/**
+	 * The Constant HBASE_ZOOKEEPER_QUORUM.
+	 */
+	private static final String HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum";
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(PrepareCopyTableJobNode.class);
+
+	/**
+	 * The source table.
+	 */
+	private String sourceCluster;
+
+	/**
+	 * The target cluster.
+	 */
+	private String targetCluster;
+
+	private String sourceTable;
+
+	private String targetTable;
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	/*
+	 * (non-Javadoc)
+	 *
+	 * @see eu.dnetlib.msro.workflows.nodes.SimpleJobNode#execute(com.googlecode.sarasvati.NodeToken)
+	 */
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		checkNodeParams();
+
+		final String outputQuorum = getOutputQuorum();
+		log.info("build hbase quorum: " + outputQuorum);
+		env.setAttribute("peer.adr", outputQuorum);
+		env.setAttribute("sourceCluster", getSourceCluster());
+		env.setAttribute("sourceTable", getSourceTable());
+
+		env.setAttribute("targetCluster", getTargetCluster());
+		env.setAttribute("targetTable", getTargetTable());
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	/**
+	 * Builds the output quorum.
+	 *
+	 * @return the output quorum
+	 * @throws HadoopServiceException when cannot retrieve the clustr configuration
+	 * @throws MSROException          when some of the needed properties is missing in the cluster configuration
+	 */
+	private String getOutputQuorum() throws HadoopServiceException, MSROException {
+		final Map<String, String> conf = this.serviceLocator.getService(HadoopService.class).getClusterConfiguration(getTargetCluster());
+		log.debug(conf);
+
+		final String hbaseQuorum = conf.get(HBASE_ZOOKEEPER_QUORUM);
+		final String hbasePort = conf.get(HBASE_ZOOKEEPER_CLIENT_PORT);
+		final String znodeParent = conf.get(ZOOKEEPER_ZNODE_PARENT);
+
+		checkParamExist(hbaseQuorum, String.format("unable to find property '%s' in cluster configuration: %s", HBASE_ZOOKEEPER_QUORUM, hbaseQuorum));
+		checkParamExist(hbasePort, String.format("unable to find property '%s' in cluster configuration: %s", HBASE_ZOOKEEPER_CLIENT_PORT, hbasePort));
+		checkParamExist(znodeParent, String.format("unable to find property '%s' in cluster configuration: %s", ZOOKEEPER_ZNODE_PARENT, znodeParent));
+
+		final String outputQuorum = String.format("%s:%s:%s", hbaseQuorum, hbasePort, znodeParent);
+		return outputQuorum;
+	}
+
+	/**
+	 * Checks the wf params.
+	 *
+	 * @throws MSROException          the MSRO exception
+	 * @throws HadoopServiceException
+	 */
+	private void checkNodeParams() throws MSROException, HadoopServiceException {
+
+		checkParamExist(getSourceCluster(), "source cluster must be set");
+		checkParamExist(getTargetCluster(), "target cluster must be set");
+		checkParamExist(getSourceTable(), "source table must be set");
+		checkParamExist(getTargetTable(), "target table must be set");
+
+		final HadoopService hadoop = this.serviceLocator.getService(HadoopService.class);
+
+		final Set<String> clusters = Sets.newHashSet(hadoop.listClusters());
+		if (!clusters.contains(getSourceCluster())) { throw new MSROException(String.format("source cluster '%s' doesn not exists", getSourceCluster())); }
+		if (!clusters.contains(getTargetCluster())) { throw new MSROException(String.format("target cluster '%s' doesn not exists", getTargetCluster())); }
+
+		if (!hadoop.existHbaseTable(getSourceCluster(), getSourceTable())) {
+			throw new MSROException(String.format(
+					"source table '%s' doesn not exists on cluster '%s'", getSourceTable(), getSourceCluster()));
+		}
+	}
+
+	/**
+	 * Check parameter existence.
+	 *
+	 * @param param the param
+	 * @param msg   the msg
+	 * @throws MSROException the MSRO exception
+	 */
+	private void checkParamExist(final String param, final String msg) throws MSROException {
+		if (StringUtils.isBlank(param)) { throw new MSROException(msg); }
+	}
+
+	public String getSourceCluster() {
+		return this.sourceCluster;
+	}
+
+	public void setSourceCluster(final String sourceCluster) {
+		this.sourceCluster = sourceCluster;
+	}
+
+	public String getTargetCluster() {
+		return this.targetCluster;
+	}
+
+	public void setTargetCluster(final String targetCluster) {
+		this.targetCluster = targetCluster;
+	}
+
+	public String getSourceTable() {
+		return this.sourceTable;
+	}
+
+	public void setSourceTable(final String sourceTable) {
+		this.sourceTable = sourceTable;
+	}
+
+	public String getTargetTable() {
+		return this.targetTable;
+	}
+
+	public void setTargetTable(final String targetTable) {
+		this.targetTable = targetTable;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/MDStoreBatchExporterJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/MDStoreBatchExporterJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/MDStoreBatchExporterJobNode.java	(revision 43444)
@@ -0,0 +1,81 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import javax.annotation.Resource;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.common.ResultSet;
+import eu.dnetlib.rmi.data.MDStoreService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * This jobNode reads all the metadata records from all the mongodb collections and generates an hdfs sequence file.
+ *
+ * @author claudio
+ */
+public class MDStoreBatchExporterJobNode extends SimpleJobNode {
+
+	private static final Log log = LogFactory.getLog(MDStoreBatchExporterJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	private String format;
+
+	private String layout;
+
+	private String interpretation;
+
+	private String outputEprParam;
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	@Override
+	public String execute(final Env env) throws Exception {
+		log.info("bulk reading MDStores: " + getFormat() + "-" + getLayout() + "-" + getInterpretation());
+		final ResultSet<String> rs =
+				getServiceLocator().getService(MDStoreService.class).bulkDeliverMDRecords(getFormat(), getLayout(), getInterpretation());
+
+		env.setAttribute(getOutputEprParam(), rs);
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getFormat() {
+		return format;
+	}
+
+	public void setFormat(final String format) {
+		this.format = format;
+	}
+
+	public String getLayout() {
+		return layout;
+	}
+
+	public void setLayout(final String layout) {
+		this.layout = layout;
+	}
+
+	public String getInterpretation() {
+		return interpretation;
+	}
+
+	public void setInterpretation(final String interpretation) {
+		this.interpretation = interpretation;
+	}
+
+	public String getOutputEprParam() {
+		return outputEprParam;
+	}
+
+	public void setOutputEprParam(final String outputEprParam) {
+		this.outputEprParam = outputEprParam;
+	}
+
+	public UniqueServiceLocator getServiceLocator() {
+		return serviceLocator;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/dedup/queryDissimilarities.sql
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/dedup/queryDissimilarities.sql	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/dedup/queryDissimilarities.sql	(revision 43444)
@@ -0,0 +1 @@
+SELECT e.entitytype, d.id1, d.id2 FROM dissimilarities d JOIN entities e on (d.id1 = e.id)
\ No newline at end of file
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/dedup/querySimilaritiesBySet.sql.st
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/dedup/querySimilaritiesBySet.sql.st	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/dedup/querySimilaritiesBySet.sql.st	(revision 43444)
@@ -0,0 +1,6 @@
+SELECT * FROM (
+	SELECT g.id, date, entitytype, array_agg(sg.objidentifier) as group 
+	FROM groups g LEFT OUTER join similarity_groups sg ON (g.id = sg.groupid) 
+	WHERE g.actionsetid = '%s'
+	GROUP BY g.id, date, entitytype) AS T 
+WHERE array_length(T.group, 1) > 1
\ No newline at end of file
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/prepareQueryDatasources.sql
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/prepareQueryDatasources.sql	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/prepareQueryDatasources.sql	(revision 43444)
@@ -0,0 +1,15 @@
+CREATE OR REPLACE VIEW temp_compliances AS 
+	SELECT 
+		a.id                 as api, 
+		a.compatibilityclass as compliance
+	FROM 
+		api a
+	WHERE
+		a.id NOT IN (SELECT api from apicollections WHERE param = 'overriding_compliance')
+	UNION SELECT
+		a.id as api, 
+		ac.original as compliance
+	FROM
+		api a LEFT OUTER JOIN apicollections ac ON (ac.api = a.id) 
+	WHERE
+		ac.param = 'overriding_compliance';
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/CreateHBaseTableJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/CreateHBaseTableJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/CreateHBaseTableJobNode.java	(revision 43444)
@@ -0,0 +1,49 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import java.util.Set;
+
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.data.hadoop.HadoopService;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class CreateHBaseTableJobNode extends AbstractHBaseAdminJobNode {
+
+	private static final Log log = LogFactory.getLog(CreateHBaseTableJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	private boolean reuseRegionInfo = false;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+		final Set<String> columns = getColumns(env);
+
+		log.info("Ensuring table " + getTableName() + " on cluster: '" + getCluster() + "' - columns: " + columns);
+
+		final HadoopService hadoopService = getServiceLocator().getService(HadoopService.class);
+		log.info(String.format("reuse region info: %s", isReuseRegionInfo()));
+		if (isReuseRegionInfo()) {
+			final String jsonConf = env.getAttribute(getTableConfigurationParamName(), String.class);
+			log.debug(String.format("region info: %s", jsonConf));
+			if (StringUtils.isBlank(jsonConf)) {
+				throw new MSROException("cannot find HBase table configuration in workflow env");
+			}
+
+			hadoopService.createConfiguredHbaseTable(getCluster(), getTableName(), jsonConf);
+		} else {
+			hadoopService.createHbaseTable(getCluster(), getTableName(), columns);
+		}
+		return Arc.DEFAULT_ARC;
+	}
+
+	public boolean isReuseRegionInfo() {
+		return reuseRegionInfo;
+	}
+
+	public void setReuseRegionInfo(final boolean reuseRegionInfo) {
+		this.reuseRegionInfo = reuseRegionInfo;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/StoreHdfsRecordsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/StoreHdfsRecordsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/StoreHdfsRecordsJobNode.java	(revision 43444)
@@ -0,0 +1,83 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import java.util.Map;
+
+import eu.dnetlib.enabling.resultset.client.ResultSetClient;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import eu.dnetlib.msro.workflows.nodes.blackboard.BlackboardWorkflowJobListener;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+import eu.dnetlib.rmi.common.ResultSet;
+import eu.dnetlib.rmi.data.hadoop.HadoopBlackboardActions;
+import eu.dnetlib.rmi.data.hadoop.HadoopService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class StoreHdfsRecordsJobNode extends BlackboardJobNode {
+
+	private static final Log log = LogFactory.getLog(StoreHdfsRecordsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	private String inputEprParam;
+	private String hdfsPath;
+	private String cluster;
+
+	@Autowired
+	private ResultSetClient resultSetClient;
+
+	@Override
+	protected String obtainServiceId(final Env env) {
+		return getServiceLocator().getServiceId(HadoopService.class);
+	}
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		log.info("Invoking blackboard method");
+
+		final ResultSet<?> rs = token.getEnv().getAttribute(inputEprParam, ResultSet.class);
+		token.setProgressProvider(new ResultsetProgressProvider(rs, resultSetClient));
+
+		job.setAction(HadoopBlackboardActions.IMPORT_EPR_HDFS.toString());
+		job.getParameters().put("input_epr", rs.toJson());
+		job.getParameters().put("path", getHdfsPath());
+		job.getParameters().put("cluster", getCluster());
+	}
+
+	@Override
+	protected BlackboardWorkflowJobListener generateBlackboardListener(final Token token) {
+		return new BlackboardWorkflowJobListener(token) {
+
+			@Override
+			protected void responseToEnv(final Env env, final Map<String, String> responseParams) {
+				env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + getNodeName() + ":count", responseParams.get("count"));
+			}
+		};
+	}
+
+	public String getInputEprParam() {
+		return inputEprParam;
+	}
+
+	public void setInputEprParam(final String inputEprParam) {
+		this.inputEprParam = inputEprParam;
+	}
+
+	public String getCluster() {
+		return cluster;
+	}
+
+	public void setCluster(final String cluster) {
+		this.cluster = cluster;
+	}
+
+	public String getHdfsPath() {
+		return hdfsPath;
+	}
+
+	public void setHdfsPath(final String hdfsPath) {
+		this.hdfsPath = hdfsPath;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/CreateHdfsDirectoryJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/CreateHdfsDirectoryJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/CreateHdfsDirectoryJobNode.java	(revision 43444)
@@ -0,0 +1,55 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.rmi.data.hadoop.HadoopBlackboardActions;
+import eu.dnetlib.rmi.data.hadoop.HadoopService;
+
+public class CreateHdfsDirectoryJobNode extends BlackboardJobNode {
+
+	private String cluster;
+
+	private boolean force = false;
+
+	private String path;
+
+	@Override
+	protected String obtainServiceId(final Env env) {
+		return getServiceLocator().getServiceId(HadoopService.class);
+	}
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+
+		job.setAction(HadoopBlackboardActions.CREATE_HDFS_DIR.toString());
+		job.getParameters().put("cluster", getCluster());
+		job.getParameters().put("force", String.valueOf(isForce()));
+		job.getParameters().put("path", getPath());
+	}
+
+	public String getCluster() {
+		return cluster;
+	}
+
+	public void setCluster(final String cluster) {
+		this.cluster = cluster;
+	}
+
+	public boolean isForce() {
+		return force;
+	}
+
+	public void setForce(boolean force) {
+		this.force = force;
+	}
+
+	public String getPath() {
+		return path;
+	}
+
+	public void setPath(final String path) {
+		this.path = path;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/prepareQueryDefault.sql
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/prepareQueryDefault.sql	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/prepareQueryDefault.sql	(revision 43444)
@@ -0,0 +1 @@
+UPDATE policies SET name = 'xx' where 1 = 2
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryProjects.sql
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryProjects.sql	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryProjects.sql	(revision 43444)
@@ -0,0 +1,93 @@
+--WITH RECURSIVE fundingtree(id, parent_id, payload, depth) 
+--AS (
+--	SELECT  f.id, 
+--		ff.funding1, 
+--		'{"funding_level_0":{"class":"' || f.semanticclass || '", "id":"' || f.id || '", "description":"' || f.description || '", "name":"' || f.name || '", "parent":{}}}', 
+--		1
+--		FROM fundings f LEFT OUTER JOIN funding_funding ff ON f.id = ff.funding1 
+--		WHERE ff.funding2 IS NULL
+--	UNION 
+--	SELECT  f1.id, 
+--		ff.funding2,
+--		'{"funding_level_'||depth||'":{"class":"' || f1.semanticclass || '", "id":"' || f1.id || '", "description":"' || f1.description || '", "name":"' || f1.name || '", "parent":'|| ft.payload ||'}}',
+--		ft.depth + 1
+--		FROM funding_funding ff, fundingtree ft, fundings f2, fundings f1
+--		WHERE ft.id = ff.funding2 AND f2.id = ft.id AND ff.funding1 = f1.id AND depth <= 10)		
+
+SELECT 	p.id 					                    as projectid,
+		p.code										as code,
+		p.websiteurl               					as websiteurl,		
+		p.acronym                  					as acronym,
+		p.title                    					as title,
+		p.startdate									as startdate,
+		p.enddate                 					as enddate,
+		p.call_identifier          					as callidentifier,
+		p.keywords									as keywords,		
+		p.duration									as duration,
+		p.ec_sc39				                    as ecsc39,
+		p.oa_mandate_for_publications               as oamandatepublications,
+		p.ec_article29_3                            as ecarticle29_3,
+		p.dateofcollection							as dateofcollection,
+		p.inferred									as inferred,
+		p.deletedbyinference						as deletedbyinference,
+		p.trust										as trust,
+		p.inferenceprovenance						as inferenceprovenance,
+		dc.id										as collectedfromid,
+		dc.officialname								as collectedfromname,
+		
+		cc.code || '@@@' || cc.name || '@@@' || cs.code || '@@@' || cs.name	as contracttype,
+		pac.code || '@@@' || pac.name || '@@@' || pas.code || '@@@' || pas.name	as provenanceaction,
+		
+		array_agg(DISTINCT i.pid  || '###' || idc.code || '@@@' || idc.name || '@@@' || ids.code || '@@@' || ids.name)	as pid,
+		array_agg(DISTINCT s.name || '###' || sc.code  || '@@@' || sc.name  || '@@@' || ss.code  || '@@@' || ss.name)   as subjects,
+		
+		array_agg(fp.path)		                as fundingtree
+	
+FROM projects p
+	left outer join class cc on (cc.code = p.contracttypeclass)	
+	left outer join scheme cs on (cs.code = p.contracttypescheme)
+	
+	left outer join class pac on (pac.code = p.provenanceactionclass)	
+	left outer join scheme pas on (pas.code = p.provenanceactionscheme)		
+	
+	left outer join projectpids pp on (pp.project = p.id)
+	left outer join identities i on (i.pid = pp.pid)
+	left outer join class idc on (idc.code = i.issuertypeclass)
+	left outer join scheme ids on (ids.code = i.issuertypescheme)
+	
+	left outer join datasources dc on (dc.id = p.collectedfrom)
+	
+	left outer join project_fundingpath pf on (pf.project = p.id)
+	left outer join fundingpaths fp on (fp.id = pf.funding)
+	
+	left outer join project_subject ps on (ps.project = p.id)
+	left outer join subjects s on (s.id = ps.subject)
+	
+	left outer join class sc on (sc.code = s.semanticclass)
+	left outer join scheme ss on (ss.code = s.semanticscheme)
+	
+GROUP BY 	
+		p.id,
+		p.code,
+		p.websiteurl,		
+		p.acronym,
+		p.title,
+		p.startdate,
+		p.enddate,
+		p.call_identifier,
+		p.keywords,		
+		p.duration,
+		p.ec_sc39,
+		p.oa_mandate_for_publications,
+		p.ec_article29_3,
+		p.dateofcollection,
+		p.inferred,
+		p.deletedbyinference,
+		p.trust,
+		p.inferenceprovenance,
+		dc.id,
+		dc.officialname,		
+		cc.code, cc.name, cs.code, cs.name,
+		pac.code, pac.name, pas.code, pas.name
+		
+		
\ No newline at end of file
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/datacite/SplitterDatasetsIterator.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/datacite/SplitterDatasetsIterator.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/datacite/SplitterDatasetsIterator.java	(revision 43444)
@@ -0,0 +1,147 @@
+package eu.dnetlib.msro.workflows.nodes.datacite;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathConstants;
+import javax.xml.xpath.XPathFactory;
+
+import com.google.common.collect.Maps;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.w3c.dom.*;
+
+/**
+ * The Class SplitterDatasetsIterator.
+ */
+public class SplitterDatasetsIterator {
+
+	/**
+	 * The Constant log.
+	 */
+	private static final Log log = LogFactory.getLog(SplitterDatasetsIterator.class);
+
+	/**
+	 * The end queue.
+	 */
+	public static String END_QUEUE = "END_QUEUE";
+
+	/**
+	 * The publications.
+	 */
+	private Queue<String> publications;
+
+	/**
+	 * The input epr.
+	 */
+	private Iterable<String> inputEPR;
+
+	/**
+	 * The root name.
+	 */
+	private String rootName;
+
+	/**
+	 * Instantiates a new splitter datasets iterator.
+	 *
+	 * @param datasetsQueue     the datasets queue
+	 * @param publicationsQueue the publications queue
+	 * @param inputEPR          the input epr
+	 * @param rootName          the root name
+	 */
+	public SplitterDatasetsIterator(final Queue<String> publicationsQueue, final Iterable<String> inputEPR, final String rootName) {
+		this.publications = publicationsQueue;
+		this.inputEPR = inputEPR;
+		this.rootName = rootName;
+
+	}
+
+	/**
+	 * Populate queues.
+	 */
+	public void populateQueues() {
+
+		if (this.inputEPR == null) return;
+		for (String inputXML : inputEPR) {
+
+			List<String> publication_extracted = extractByTag(new ByteArrayInputStream(inputXML.getBytes(StandardCharsets.UTF_8)), "publication", inputXML);
+
+			if (publication_extracted != null) {
+				publications.addAll(publication_extracted);
+
+			}
+		}
+		publications.add(END_QUEUE);
+
+	}
+
+	/**
+	 * Extract by tag.
+	 *
+	 * @param inputXML the input xml
+	 * @param tag      the tag
+	 * @return the list
+	 */
+	private List<String> extractByTag(final InputStream inputXML, final String tag, final String input) {
+		try {
+
+			DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+			Document doc = dbf.newDocumentBuilder().parse(inputXML);
+
+			XPath xpath = XPathFactory.newInstance().newXPath();
+
+			Node rootNode = (Node) xpath.evaluate("//*[local-name()='" + this.rootName + "']", doc, XPathConstants.NODE);
+
+			NamedNodeMap attributes = rootNode.getAttributes();
+
+			Map<String, String> nameSpaces = Maps.newHashMap();
+
+			for (int i = 0; i < attributes.getLength(); i++) {
+				Node node = attributes.item(i);
+				String name = node.getNodeName();
+				if (name.startsWith("xmlns:")) {
+					nameSpaces.put(StringUtils.substringAfter(name, "xmlns:"), node.getNodeValue());
+				}
+
+			}
+			xpath = XPathFactory.newInstance().newXPath();
+			NodeList nodes = (NodeList) xpath.evaluate("//*[local-name()='" + tag + "']/*[local-name()='record']", doc, XPathConstants.NODESET);
+
+			if ((nodes != null) && (nodes.getLength() > 0)) {
+				List<String> result = new ArrayList<String>();
+				for (int i = 0; i < nodes.getLength(); i++) {
+					Document currentDoc = dbf.newDocumentBuilder().newDocument();
+					Node imported = currentDoc.importNode(nodes.item(i), true);
+					for (String key : nameSpaces.keySet()) {
+						Element element = (Element) imported;
+						element.setAttributeNS(XMLConstants.XMLNS_ATTRIBUTE_NS_URI, "xmlns:" + key, nameSpaces.get(key));
+					}
+					Transformer transformer = TransformerFactory.newInstance().newTransformer();
+					DOMSource mydoc = new DOMSource(imported);
+					StringWriter writer = new StringWriter();
+					transformer.transform(mydoc, new StreamResult(writer));
+					String record = writer.toString();
+					result.add(record);
+				}
+				return result;
+			}
+		} catch (Exception e) {
+			log.error("Error on extracting " + tag, e);
+			return null;
+		}
+		return null;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/openaire/api/objects/DatasourceEntry.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/openaire/api/objects/DatasourceEntry.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/openaire/api/objects/DatasourceEntry.java	(revision 43444)
@@ -0,0 +1,57 @@
+package eu.dnetlib.openaire.api.objects;
+
+import eu.dnetlib.miscutils.functional.hash.Hashing;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * Created by michele on 02/12/15.
+ */
+public class DatasourceEntry {
+
+	private String id;
+	private String name;
+	private String prefix;
+
+	public DatasourceEntry() {
+	}
+
+	public DatasourceEntry(final String id, final String name, final String prefix) {
+		this.id = id;
+		this.name = name;
+		this.prefix = prefix;
+	}
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(final String name) {
+		this.name = name;
+	}
+
+	public String getPrefix() {
+		return prefix;
+	}
+
+	public void setPrefix(final String prefix) {
+		this.prefix = prefix;
+	}
+
+	public String calculateOpenaireId() {
+		if (StringUtils.isNotBlank(id)) {
+			final String[] arr = id.split("::");
+			if (arr.length == 2) {
+				return String.format("%s::%s", arr[0], Hashing.md5(arr[1]));
+			}
+		}
+		return "";
+	}
+}
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/repository/CMR_EntityRegistry.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/repository/CMR_EntityRegistry.xml	(revision 43443)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/repository/CMR_EntityRegistry.xml	(nonexistent)
@@ -1,68 +0,0 @@
-<RESOURCE_PROFILE>
-	<HEADER>
-		<RESOURCE_IDENTIFIER value="1b950898-839b-47f8-a5f5-148b84b453bc_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU="/>
-		<RESOURCE_TYPE value="RepositoryServiceResourceType"/>
-		<RESOURCE_KIND value="RepositoryServiceResources"/>
-		<RESOURCE_URI value=""/>
-		<DATE_OF_CREATION value="2016-03-24T12:40:17+01:00"/>
-		<PROTOCOL/>
-	</HEADER>
-	<BODY>
-		<CONFIGURATION>
-			<DATASOURCE_TYPE>wds:entityregistry</DATASOURCE_TYPE>
-			<DATASOURCE_ORIGINAL_ID provenance="WDS">CMREntityRegistry</DATASOURCE_ORIGINAL_ID>
-			<DATASOURCE_AGGREGATED>false</DATASOURCE_AGGREGATED>
-			<ENVIRONMENTS>
-				<ENVIRONMENT>WDS</ENVIRONMENT>
-			</ENVIRONMENTS>
-			<TYPOLOGY/>
-			<MAX_SIZE_OF_DATASTRUCTURE>0</MAX_SIZE_OF_DATASTRUCTURE>
-			<AVAILABLE_DISKSPACE>0</AVAILABLE_DISKSPACE>
-			<MAX_NUMBER_OF_DATASTRUCTURE>0</MAX_NUMBER_OF_DATASTRUCTURE>
-			<OFFICIAL_NAME>Common Metadata Repository (CMR)</OFFICIAL_NAME>
-			<ENGLISH_NAME>Common Metadata Repository (CMR)</ENGLISH_NAME>
-			<ICON_URI></ICON_URI>
-			<COUNTRY/>
-			<LOCATION>
-				<LONGITUDE>0.0</LONGITUDE>
-				<LATITUDE>0.0</LATITUDE>
-				<TIMEZONE>1.0</TIMEZONE>
-			</LOCATION>
-			<REPOSITORY_WEBPAGE>https://cmr.earthdata.nasa.gov/search/</REPOSITORY_WEBPAGE>
-			<REPOSITORY_INSTITUTION/>
-			<ADMIN_INFO>sandro.labruzzo@isti.cnr.it</ADMIN_INFO>
-			<INTERFACES>
-				<INTERFACE active="false" compliance="efg" contentDescription="metadata" id="api_________::barch::0" label="content provider" removable="false" typology="efg:contentprovider">
-					<ACCESS_PROTOCOL extensions="wds">CMRDatasource</ACCESS_PROTOCOL>
-					<BASE_URL></BASE_URL>
-					<INTERFACE_EXTRA_FIELD name="metadata_identifier_path">//*[local-name()='header']/*[local-name()='identifier']</INTERFACE_EXTRA_FIELD>
-				</INTERFACE>
-			</INTERFACES>
-			<EXTRA_FIELDS>
-				<FIELD>
-					<key>DataSourceId</key>
-					<value>cmr_entity_registry</value>
-				</FIELD>
-				<FIELD>
-					<key>NamespacePrefix</key>
-					<value>cmr_entity_registry</value>
-				</FIELD>
-				<FIELD>
-					<key>VERIFIED</key>
-					<value>NO</value>
-				</FIELD>
-			</EXTRA_FIELDS>
-			<REGISTERED_BY>sandro.labruzzo@isti.cnr.it</REGISTERED_BY>
-		</CONFIGURATION>
-		<STATUS>
-			<NUMBER_OF_OBJECTS>0</NUMBER_OF_OBJECTS>
-		</STATUS>
-		<QOS>
-			<AVAILABILITY>0</AVAILABILITY>
-			<CAPACITY/>
-			<THROUGHPUT>0.0</THROUGHPUT>
-		</QOS>
-		<SECURITY_PARAMETERS/>
-		<BLACKBOARD/>
-	</BODY>
-</RESOURCE_PROFILE>
\ No newline at end of file
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/db/queryClaimsDmf.sql
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/db/queryClaimsDmf.sql	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/db/queryClaimsDmf.sql	(revision 43444)
@@ -0,0 +1 @@
+SELECT regexp_replace(xml, '&(?!amp;)', '&amp;', 'g') as xml, provenance FROM claims WHERE type = 'dmf2actions' and set = 'userclaim_dmf' and provenance <> 'user:claim:datacite'
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/index/openaireLayoutToRecordStylesheet.xsl
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/index/openaireLayoutToRecordStylesheet.xsl	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/index/openaireLayoutToRecordStylesheet.xsl	(revision 43444)
@@ -0,0 +1,116 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                xmlns:nxsl="http://www.w3.org/1999/XSL/TransformXX" version="1.0"
+>
+
+	<xsl:output omit-xml-declaration="yes" method="xml"/>
+	<xsl:namespace-alias stylesheet-prefix="nxsl" result-prefix="xsl"/>
+
+	<xsl:param name="format"/>
+
+	<xsl:template match="/">
+		<xsl:apply-templates select="//LAYOUT"/>
+	</xsl:template>
+
+	<xsl:template match="LAYOUT">
+		<nxsl:stylesheet xmlns:dnet="eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions"
+		                 version="1.0"
+		                 extension-element-prefixes="exsl" exclude-result-prefixes="dnet">
+
+			<nxsl:output version="1.0" omit-xml-declaration="yes"
+			             method="xml"/>
+
+			<nxsl:variable name="format">
+				<xsl:value-of select="$format"/>
+			</nxsl:variable>
+
+			<nxsl:template match="/">
+				<indexRecord>
+					<indexRecordIdentifier>
+						<nxsl:value-of select="//dri:objIdentifier"/>
+					</indexRecordIdentifier>
+					<targetFields>
+						<nxsl:if test="count(//*[local-name()='metadata']/*) &gt; 0">
+							<xsl:apply-templates select="FIELDS/FIELD[@indexable='true']"/>
+						</nxsl:if>
+					</targetFields>
+					<dnetResult>
+						<nxsl:copy-of select="/*[local-name()='record']/*[local-name()='result']/*[local-name()='header']"/>
+						<metadata>
+							<xsl:apply-templates select="FIELDS/FIELD"
+							                     mode="result"/>
+						</metadata>
+					</dnetResult>
+				</indexRecord>
+			</nxsl:template>
+		</nxsl:stylesheet>
+	</xsl:template>
+
+	<xsl:template match="FIELD[@indexable='true']">
+		<xsl:choose>
+			<xsl:when test="@constant">
+				<xsl:element name="{@name}">
+					<xsl:value-of select="@constant"/>
+				</xsl:element>
+			</xsl:when>
+			<xsl:when test="@value and not(@xpath)">
+				<xsl:choose>
+					<xsl:when test="@type='date'">
+						<nxsl:variable name="{@name}" select="dnet:normalizeDate(normalize-space({normalize-space(@value)}), false())"/>
+						<nxsl:if test="string-length(${@name}) &gt; 0">
+							<nxsl:element name="{@name}">
+								<nxsl:value-of select="${@name}"/>
+							</nxsl:element>
+						</nxsl:if>
+					</xsl:when>
+					<xsl:otherwise>
+						<nxsl:element name="{@name}">
+							<nxsl:value-of select="{@value}"/>
+						</nxsl:element>
+					</xsl:otherwise>
+				</xsl:choose>
+			</xsl:when>
+			<xsl:otherwise>
+				<xsl:variable name="value">
+					<xsl:choose>
+						<xsl:when test="@value">
+							<xsl:value-of select="@value"/>
+						</xsl:when>
+						<xsl:otherwise>
+							.
+						</xsl:otherwise>
+					</xsl:choose>
+				</xsl:variable>
+				<nxsl:for-each select="{@xpath}">
+					<xsl:element name="{@name}">
+						<xsl:if test="@external='true'">
+							<xsl:attribute name="external">true</xsl:attribute>
+						</xsl:if>
+						<xsl:choose>
+							<xsl:when test="@tokenizable='false'">
+								<nxsl:value-of select="normalize-space({normalize-space($value)})"/>
+							</xsl:when>
+							<xsl:otherwise>
+								<nxsl:value-of select="{normalize-space($value)}"/>
+							</xsl:otherwise>
+						</xsl:choose>
+					</xsl:element>
+				</nxsl:for-each>
+			</xsl:otherwise>
+		</xsl:choose>
+	</xsl:template>
+
+
+	<xsl:template match="FIELD" mode="result">
+		<xsl:if test="@result='true'">
+			<nxsl:copy-of select="{@xpath}"/>
+		</xsl:if>
+	</xsl:template>
+
+	<xsl:template match="FIELD" mode="header">
+		<xsl:if test="@header='true'">
+			<nxsl:copy-of select="{@xpath}"/>
+		</xsl:if>
+	</xsl:template>
+
+</xsl:stylesheet>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/openaire/applicationContext-openaire.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/openaire/applicationContext-openaire.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/openaire/applicationContext-openaire.xml	(revision 43444)
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:p="http://www.springframework.org/schema/p"
+       xmlns="http://www.springframework.org/schema/beans"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+	<bean id="recentPublicationsQueue" class="eu.dnetlib.openaire.api.RecentPublicationsQueue"
+	      p:db-ref="msroApiMongoDB" p:collection="${openaireplus.msro.api.mongo.collection}"
+	      init-method="init"/>
+
+	<bean id="apiMongoServer" class="com.mongodb.Mongo">
+		<constructor-arg index="0" type="com.mongodb.ServerAddress">
+			<bean class="com.mongodb.ServerAddress">
+				<constructor-arg index="0"
+				                 value="${openaireplus.msro.api.mongo.host}"/>
+				<constructor-arg index="1"
+				                 value="${openaireplus.msro.api.mongo.port}"/>
+			</bean>
+		</constructor-arg>
+	</bean>
+
+	<bean id="apiOafToIndexRecordFactory" class="eu.dnetlib.msro.workflows.nodes.index.OafToIndexRecordFactory"
+	      p:layoutToRecord="${openaireplus.msro.api.layoutToRecord.xslt}"/>
+
+
+	<bean id="msroApiMongoDB" factory-bean="apiMongoServer"
+	      factory-method="getDB">
+		<constructor-arg index="0" value="${openaireplus.msro.api.mongo.db}"/>
+	</bean>
+
+	<bean id="openaireplusApisVelocityEngine" class="org.springframework.ui.velocity.VelocityEngineFactoryBean">
+		<property name="velocityProperties">
+			<value>
+				resource.loader=class
+				class.resource.loader.class=org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader
+				runtime.log.logsystem.class=org.apache.velocity.runtime.log.Log4JLogChute
+			</value>
+		</property>
+	</bean>
+
+</beans>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/openaire/api/applicationContext-api.properties
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/openaire/api/applicationContext-api.properties	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/openaire/api/applicationContext-api.properties	(revision 43444)
@@ -0,0 +1,7 @@
+openaireplus.msro.api.mongo.host=localhost
+openaireplus.msro.api.mongo.port=27017
+openaireplus.msro.api.mongo.db=openaireplus_apis
+openaireplus.msro.api.mongo.collection=recent_publications
+openaireplus.msro.api.layoutToRecord.xslt=/eu/dnetlib/msro/openaireplus/workflows/index/openaireLayoutToRecordStylesheet.xsl
+openaireplus.msro.api.findSolrIndexUrl.xquery=/eu/dnetlib/msro/openaireplus/api/findSolrIndexUrl.xquery
+openaireplus.msro.api.findIndexDsInfo.xquery=/eu/dnetlib/msro/openaireplus/api/findIndexDsInfo.xquery
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/PrepareStatsParamJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/PrepareStatsParamJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/PrepareStatsParamJobNode.java	(revision 43444)
@@ -0,0 +1,50 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import java.util.Map;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.data.hadoop.ClusterName;
+import eu.dnetlib.rmi.data.hadoop.HadoopService;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class PrepareStatsParamJobNode extends SimpleJobNode {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(PrepareStatsParamJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	private static final String xqueryIsLookupEndpoint =
+			"//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='IS_LookUpServiceResourceType']//PROTOCOL[./@name='SOAP']/@address/string()";
+
+	@Autowired
+	private UniqueServiceLocator serviceLocator;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+
+		log.info("prepare stats update job");
+
+		final Map<String, String> conf = serviceLocator.getService(HadoopService.class).getClusterConfiguration(ClusterName.DM.toString());
+		final String nameNode = conf.get("fs.defaultFS");
+
+		env.setAttribute("nameNode", nameNode);
+		env.setAttribute("jobTracker", conf.get("mapred.job.tracker"));
+
+		env.setAttribute("isLookupEndpoint", getIsLookupEndpoint());
+
+		return Arc.DEFAULT_ARC;
+	}
+
+	private String getIsLookupEndpoint() throws ISLookUpException {
+		return this.serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xqueryIsLookupEndpoint);
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/AbstractHBaseJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/AbstractHBaseJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/AbstractHBaseJobNode.java	(revision 43444)
@@ -0,0 +1,143 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import java.io.IOException;
+import java.util.Map;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.resultset.client.ResultSetClient;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import eu.dnetlib.msro.workflows.nodes.blackboard.BlackboardWorkflowJobListener;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+import eu.dnetlib.rmi.common.ResultSet;
+import eu.dnetlib.rmi.data.hadoop.HadoopBlackboardActions;
+import eu.dnetlib.rmi.data.hadoop.HadoopService;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+/**
+ * Created by claudio on 08/04/16.
+ */
+public abstract class AbstractHBaseJobNode extends BlackboardJobNode {
+
+	private static final Log log = LogFactory.getLog(AbstractHBaseJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	private final String INPUT_HBASE_TABLE_PARAM = "hbaseTable";
+	private final String INPUT_EPR_PARAM = "input_epr";
+	private final String INPUT_CLUSTER_PARAM = "cluster";
+
+	private final String XSLT_PARAM = "xslt";
+
+	private final String OUTPUT_HBASE_TABLE_PARAM = "table";
+	private final String OUTPUT_CLUSTER_PARAM = "cluster";
+	private final String SIMULATION_PARAM = "simulation";
+
+	@Autowired
+	protected UniqueServiceLocator serviceLocator;
+
+	@Autowired
+	protected ResultSetClient resultSetClient;
+
+	protected String inputEprParam;
+
+	protected String hbaseTable;
+	protected String cluster;
+	protected String mapping;
+
+	protected boolean simulation = false;
+
+	protected abstract HadoopBlackboardActions getAction();
+
+	@Override
+	protected String obtainServiceId(final Env env) {
+		return getServiceLocator().getServiceId(HadoopService.class);
+	}
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+		log.info("Invoking blackboard method: " + getAction().toString());
+
+		final ResultSet<?> rs = token.getEnv().getAttribute(getInputEprParam(), ResultSet.class);
+		token.setProgressProvider(new ResultsetProgressProvider(rs, resultSetClient));
+
+		job.setAction(getAction().toString());
+		job.getParameters().put(INPUT_EPR_PARAM, rs.toJson());
+		job.getParameters().put(XSLT_PARAM, DnetXsltFunctions.encodeBase64(readXslt(getMapping())));
+		job.getParameters().put(OUTPUT_HBASE_TABLE_PARAM, getHbaseTable());
+		job.getParameters().put(OUTPUT_CLUSTER_PARAM, getCluster());
+		job.getParameters().put(SIMULATION_PARAM, String.valueOf(isSimulation()));
+	}
+
+	@Override
+	protected BlackboardWorkflowJobListener generateBlackboardListener(final Token token) {
+		return new BlackboardWorkflowJobListener(token) {
+
+			@Override
+			protected void responseToEnv(final Env env, final Map<String, String> responseParams) {
+				final String count = responseParams.get("count");
+				log.info(String.format("%s %s objects to HBase table %s, cluster %s", getAction().toString(), count, getHbaseTable(), getCluster()));
+				env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + getNodeName() + ":count", count);
+			}
+		};
+	}
+
+	private String readXslt(final String profileId) throws IOException, MSROException, ISLookUpException {
+		if (StringUtils.isBlank(profileId)) throw new MSROException("missing profile id");
+
+		log.info("loading mapping from profile id: " + profileId);
+
+		final String xquery =
+				String.format("/RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value='%s']/BODY/CONFIGURATION/SCRIPT/CODE/*[local-name()='stylesheet']", profileId);
+		return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xquery);
+	}
+
+	public String getInputEprParam() {
+		return inputEprParam;
+	}
+
+	public void setInputEprParam(final String inputEprParam) {
+		this.inputEprParam = inputEprParam;
+	}
+
+	public String getMapping() {
+		return mapping;
+	}
+
+	public void setMapping(final String mapping) {
+		this.mapping = mapping;
+	}
+
+	public String getCluster() {
+		return cluster;
+	}
+
+	public void setCluster(final String cluster) {
+		this.cluster = cluster;
+	}
+
+	public boolean isSimulation() {
+		return simulation;
+	}
+
+	public void setSimulation(final boolean simulation) {
+		this.simulation = simulation;
+	}
+
+	public String getHbaseTable() {
+		return hbaseTable;
+	}
+
+	public void setHbaseTable(final String hbaseTable) {
+		this.hbaseTable = hbaseTable;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/ReuseHdfsRecordsJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/ReuseHdfsRecordsJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/ReuseHdfsRecordsJobNode.java	(revision 43444)
@@ -0,0 +1,23 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+
+public class ReuseHdfsRecordsJobNode extends SimpleJobNode {
+
+	private boolean reuseMdRecords;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+		return String.valueOf(isReuseMdRecords());
+	}
+
+	public boolean isReuseMdRecords() {
+		return reuseMdRecords;
+	}
+
+	public void setReuseMdRecords(final boolean reuseMdRecords) {
+		this.reuseMdRecords = reuseMdRecords;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/AbstractHBaseAdminJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/AbstractHBaseAdminJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/AbstractHBaseAdminJobNode.java	(revision 43444)
@@ -0,0 +1,78 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import java.util.Set;
+import javax.annotation.Resource;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Sets;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public abstract class AbstractHBaseAdminJobNode extends SimpleJobNode {
+
+	/**
+	 * The Constant log.
+	 */
+	private static final Log log = LogFactory.getLog(AbstractHBaseAdminJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	private String tableColumnsParamName = "columns";
+	private String tableConfigurationParamName = "tableConf";
+	private String tableName;
+	private String cluster;
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	protected Set<String> getColumns(final Env env) throws MSROException {
+		String envCols = env.getAttribute(getTableColumnsParamName(), String.class);
+		if (StringUtils.isBlank(envCols)) { throw new MSROException("cannot find table description"); }
+		log.debug("using columns from env: " + envCols);
+		return Sets.newHashSet(Splitter.on(",").omitEmptyStrings().split(envCols));
+	}
+
+	protected String asCSV(final Iterable<String> columns) {
+		return Joiner.on(",").skipNulls().join(columns);
+	}
+
+	public String getCluster() {
+		return cluster;
+	}
+
+	public void setCluster(final String cluster) {
+		this.cluster = cluster;
+	}
+
+	public String getTableColumnsParamName() {
+		return tableColumnsParamName;
+	}
+
+	public void setTableColumnsParamName(final String tableColumnsParamName) {
+		this.tableColumnsParamName = tableColumnsParamName;
+	}
+
+	public UniqueServiceLocator getServiceLocator() {
+		return serviceLocator;
+	}
+
+	public String getTableConfigurationParamName() {
+		return tableConfigurationParamName;
+	}
+
+	public void setTableConfigurationParamName(final String tableConfigurationParamName) {
+		this.tableConfigurationParamName = tableConfigurationParamName;
+	}
+
+	public String getTableName() {
+		return tableName;
+	}
+
+	public void setTableName(final String tableName) {
+		this.tableName = tableName;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/DeleteHdfsPathJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/DeleteHdfsPathJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/DeleteHdfsPathJobNode.java	(revision 43444)
@@ -0,0 +1,44 @@
+package eu.dnetlib.msro.workflows.nodes.hadoop;
+
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.msro.workflows.procs.Token;
+import eu.dnetlib.rmi.data.hadoop.HadoopBlackboardActions;
+import eu.dnetlib.rmi.data.hadoop.HadoopService;
+
+public class DeleteHdfsPathJobNode extends BlackboardJobNode {
+
+	private String path;
+
+	private String cluster;
+
+	@Override
+	protected String obtainServiceId(final Env env) {
+		return getServiceLocator().getServiceId(HadoopService.class);
+	}
+
+	@Override
+	protected void prepareJob(final BlackboardJob job, final Token token) throws Exception {
+
+		job.setAction(HadoopBlackboardActions.DELETE_HDFS_PATH.toString());
+		job.getParameters().put("cluster", getCluster());
+		job.getParameters().put("path", getPath());
+	}
+
+	public String getCluster() {
+		return cluster;
+	}
+
+	public void setCluster(final String cluster) {
+		this.cluster = cluster;
+	}
+
+	public String getPath() {
+		return path;
+	}
+
+	public void setPath(final String path) {
+		this.path = path;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/OpenaireMdBuilderJobNode.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/OpenaireMdBuilderJobNode.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/OpenaireMdBuilderJobNode.java	(revision 43444)
@@ -0,0 +1,208 @@
+package eu.dnetlib.msro.workflows.nodes;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.resultset.factory.ResultSetFactory;
+import eu.dnetlib.miscutils.functional.xml.ApplyXslt;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.common.ResultSet;
+import eu.dnetlib.rmi.enabling.ISLookUpException;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import eu.dnetlib.rmi.manager.MSROException;
+import org.antlr.stringtemplate.StringTemplate;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class OpenaireMdBuilderJobNode extends SimpleJobNode {
+
+	private static final Log log = LogFactory.getLog(OpenaireMdBuilderJobNode.class);
+
+	private StringTemplate mdBuilderTemplateXslt;
+
+	private String inputEprParam;
+	private String outputEprParam;
+	private String datasourceId;
+	private String datasourceInterface;
+
+	private boolean inferred = false;
+	private boolean deletedbyinference = false;
+	private String inferenceprovenance = "";
+	private String trust = "";
+	private String provenanceactionclassname = "";
+	private String provenanceactionclassid = "";
+
+	private String overridingMetadataIdentifierXPath = "";
+
+	@Autowired
+	private ResultSetFactory resultSetFactory;
+
+	@Autowired
+	private UniqueServiceLocator serviceLocator;
+
+	@Override
+	protected String execute(final Env env) throws MSROException {
+		final ResultSet<?> inputEpr = env.getAttribute(this.inputEprParam, ResultSet.class);
+
+		if (inputEpr == null) { throw new MSROException("InputEprParam (" + this.inputEprParam + ") not found in ENV"); }
+
+		try {
+			final StringTemplate st = new StringTemplate(getMdBuilderTemplateXslt().getTemplate());
+
+			st.setAttribute("xpath", getMetadataIdentifierXPath());
+			st.setAttribute("datasourceId", this.datasourceId);
+			if (env.hasAttribute("namespacePrefix")) {
+				st.setAttribute("namespacePrefix", env.getAttribute("namespacePrefix"));
+			} else {
+				st.setAttribute("namespacePrefix", this.datasourceId);
+			}
+
+			st.setAttribute("inferred", String.valueOf(getInferred()));
+			st.setAttribute("deletedbyinference", String.valueOf(getDeletedbyinference()));
+			st.setAttribute("inferenceprovenance", getInferenceprovenance());
+			st.setAttribute("trust", getTrust());
+			st.setAttribute("provenanceactionclassname", getProvenanceactionclassname());
+			st.setAttribute("provenanceactionclassid", getProvenanceactionclassid());
+			st.setAttribute("baseurl", getBaseUrl());
+			st.setAttribute("metadatanamespace", getMetadataNamespace());
+
+			final ResultSet<String> epr = this.resultSetFactory.map(inputEpr, String.class, new ApplyXslt(st.toString()));
+
+			env.setAttribute(this.outputEprParam, epr);
+
+			return Arc.DEFAULT_ARC;
+		} catch (final ISLookUpException e) {
+			throw new MSROException("Error while initializing mdBuilder template for datasource " + this.datasourceId, e);
+		}
+	}
+
+	private String getMetadataIdentifierXPath() throws ISLookUpException {
+		final String xQuery =
+				String.format("for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType') "
+						+ "return $x//INTERFACE[@id='%s']/INTERFACE_EXTRA_FIELD[@name='metadata_identifier_path']/string()", getDatasourceInterface());
+
+		return hasOverridingMetadataIdentifierXPath() ? getOverridingMetadataIdentifierXPath() : this.serviceLocator.getService(ISLookUpService.class)
+				.getResourceProfileByQuery(xQuery);
+	}
+
+	private String getBaseUrl() throws ISLookUpException {
+		String xQuery = "/*[.//RESOURCE_IDENTIFIER/@value='{datasourceId}']//INTERFACE[@id='{interfaceId}']//BASE_URL/string()";
+		xQuery = xQuery.replace("{interfaceId}", this.datasourceInterface).replace("{datasourceId}", this.datasourceId);
+		return this.serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xQuery);
+	}
+
+	private String getMetadataNamespace() {
+		try {
+			String xQuery =
+					"let $x := /*[.//RESOURCE_IDENTIFIER/@value='{datasourceId}']//INTERFACE[@id='{interfaceId}']/ACCESS_PROTOCOL/@format/string() "
+							+ "return /*[.//RESOURCE_TYPE/@value='MetadataFormatDSResourceType']//METADATAFORMAT[@Prefix=$x]/@NameSpace/string()";
+			xQuery = xQuery.replace("{interfaceId}", this.datasourceInterface).replace("{datasourceId}", this.datasourceId);
+			return this.serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xQuery);
+		} catch (final ISLookUpException e) {
+			log.error("The interface is not OAI or the format is not found in the MetadataFormatDSResourceType, thus metadata format in the <about> section "
+					+ "cannot managed here and it will be leaved empty (for the time being)");
+			return "";
+		}
+	}
+
+	private boolean hasOverridingMetadataIdentifierXPath() {
+		return !getOverridingMetadataIdentifierXPath().isEmpty();
+	}
+
+	public String getInputEprParam() {
+		return this.inputEprParam;
+	}
+
+	public void setInputEprParam(final String inputEprParam) {
+		this.inputEprParam = inputEprParam;
+	}
+
+	public String getOutputEprParam() {
+		return this.outputEprParam;
+	}
+
+	public void setOutputEprParam(final String outputEprParam) {
+		this.outputEprParam = outputEprParam;
+	}
+
+	public String getDatasourceId() {
+		return this.datasourceId;
+	}
+
+	public void setDatasourceId(final String datasourceId) {
+		this.datasourceId = datasourceId;
+	}
+
+	public String getDatasourceInterface() {
+		return this.datasourceInterface;
+	}
+
+	public void setDatasourceInterface(final String datasourceInterface) {
+		this.datasourceInterface = datasourceInterface;
+	}
+
+	public StringTemplate getMdBuilderTemplateXslt() {
+		return this.mdBuilderTemplateXslt;
+	}
+
+	public void setMdBuilderTemplateXslt(final StringTemplate mdBuilderTemplateXslt) {
+		this.mdBuilderTemplateXslt = mdBuilderTemplateXslt;
+	}
+
+	public boolean getInferred() {
+		return this.inferred;
+	}
+
+	public void setInferred(final boolean inferred) {
+		this.inferred = inferred;
+	}
+
+	public boolean getDeletedbyinference() {
+		return this.deletedbyinference;
+	}
+
+	public void setDeletedbyinference(final boolean deletedbyinference) {
+		this.deletedbyinference = deletedbyinference;
+	}
+
+	public String getInferenceprovenance() {
+		return this.inferenceprovenance;
+	}
+
+	public void setInferenceprovenance(final String inferenceprovenance) {
+		this.inferenceprovenance = inferenceprovenance;
+	}
+
+	public String getTrust() {
+		return this.trust;
+	}
+
+	public void setTrust(final String trust) {
+		this.trust = trust;
+	}
+
+	public String getProvenanceactionclassname() {
+		return this.provenanceactionclassname;
+	}
+
+	public void setProvenanceactionclassname(final String provenanceactionclassname) {
+		this.provenanceactionclassname = provenanceactionclassname;
+	}
+
+	public String getProvenanceactionclassid() {
+		return this.provenanceactionclassid;
+	}
+
+	public void setProvenanceactionclassid(final String provenanceactionclassid) {
+		this.provenanceactionclassid = provenanceactionclassid;
+	}
+
+	public String getOverridingMetadataIdentifierXPath() {
+		return this.overridingMetadataIdentifierXPath;
+	}
+
+	public void setOverridingMetadataIdentifierXPath(final String overridingMetadataIdentifierXPath) {
+		this.overridingMetadataIdentifierXPath = overridingMetadataIdentifierXPath;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/datacite/IteratorOnQueue.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/datacite/IteratorOnQueue.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/datacite/IteratorOnQueue.java	(revision 43444)
@@ -0,0 +1,90 @@
+package eu.dnetlib.msro.workflows.nodes.datacite;
+
+import java.util.Iterator;
+import java.util.concurrent.BlockingQueue;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * The Class IteratorOnQueue.
+ *
+ * @param <T> the generic type
+ */
+public class IteratorOnQueue<T> implements Iterator<T> {
+
+	/**
+	 * The Constant log.
+	 */
+	private static final Log log = LogFactory.getLog(IteratorOnQueue.class);
+
+	/**
+	 * The input queue.
+	 */
+	private final BlockingQueue<T> inputQueue;
+
+	/**
+	 * The current item.
+	 */
+	private T currentItem;
+
+	/**
+	 * The end queue.
+	 */
+	private T endQueue;
+
+	/**
+	 * Instantiates a new iterator on queue.
+	 *
+	 * @param inputQueue the input queue
+	 * @param endQueue   the end queue
+	 */
+	public IteratorOnQueue(final BlockingQueue<T> inputQueue, final T endQueue) {
+		this.inputQueue = inputQueue;
+		this.endQueue = endQueue;
+
+		try {
+			currentItem = this.inputQueue.take();
+		} catch (InterruptedException e) {
+			log.error(e);
+		}
+	}
+
+	/**
+	 * Checks for next.
+	 *
+	 * @return true, if successful
+	 */
+	@Override
+	public boolean hasNext() {
+
+		return currentItem != this.endQueue;
+	}
+
+	/**
+	 * Next.
+	 *
+	 * @return the t
+	 */
+	@Override
+	public T next() {
+
+		T previous = currentItem;
+		try {
+			currentItem = this.inputQueue.take();
+		} catch (Exception e) {
+			log.error(e);
+		}
+		return previous;
+	}
+
+	/**
+	 * Removes the.
+	 */
+	@Override
+	public void remove() {
+		// TODO Auto-generated method stub
+
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/openaire/api/objects/PidEntry.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/openaire/api/objects/PidEntry.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/openaire/api/objects/PidEntry.java	(revision 43444)
@@ -0,0 +1,34 @@
+package eu.dnetlib.openaire.api.objects;
+
+/**
+ * Created by michele on 02/12/15.
+ */
+public class PidEntry {
+
+	private String type;
+	private String value;
+
+	public PidEntry() {
+	}
+
+	public PidEntry(final String type, final String value) {
+		this.type = type;
+		this.value = value;
+	}
+
+	public String getType() {
+		return type;
+	}
+
+	public void setType(final String type) {
+		this.type = type;
+	}
+
+	public String getValue() {
+		return value;
+	}
+
+	public void setValue(final String value) {
+		this.value = value;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/wds/collector/plugins/CMRDatasource.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/wds/collector/plugins/CMRDatasource.java	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/wds/collector/plugins/CMRDatasource.java	(revision 43444)
@@ -0,0 +1,49 @@
+package eu.dnetlib.wds.collector.plugins;
+
+import org.apache.commons.lang3.StringEscapeUtils;
+
+/**
+ * Created by sandro on 7/18/16.
+ */
+public class CMRDatasource {
+
+	private String id;
+
+	private String name;
+
+	private String url;
+
+	private String uuid;
+
+	public String getId() {
+		return StringEscapeUtils.escapeXml11(id);
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getName() {
+		return StringEscapeUtils.escapeXml11(name);
+	}
+
+	public void setName(final String name) {
+		this.name = name;
+	}
+
+	public String getUrl() {
+		return StringEscapeUtils.escapeXml11(url);
+	}
+
+	public void setUrl(final String url) {
+		this.url = url;
+	}
+
+	public String getUuid() {
+		return StringEscapeUtils.escapeXml11(uuid);
+	}
+
+	public void setUuid(final String uuid) {
+		this.uuid = uuid;
+	}
+}
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/dedup/queryDissimilaritiesBySet.sql.st
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/dedup/queryDissimilaritiesBySet.sql.st	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/dedup/queryDissimilaritiesBySet.sql.st	(revision 43444)
@@ -0,0 +1,2 @@
+SELECT e.entitytype, d.id1, d.id2 from dissimilarities d join entities e on (d.id1 = e.id)
+WHERE d.actionsetid = '%s'
\ No newline at end of file
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryClaimsUpdate.sql
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryClaimsUpdate.sql	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryClaimsUpdate.sql	(revision 43444)
@@ -0,0 +1,7 @@
+SELECT
+	regexp_replace(xml, '<\?xml version="1\.0" encoding="UTF-8"\?>', '', 'i') AS xml,
+	provenance
+FROM claims
+WHERE type = 'updates2actions' AND set = 'userclaim_dmf'
+
+
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryDatasourceOrganization.sql
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryDatasourceOrganization.sql	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryDatasourceOrganization.sql	(revision 43444)
@@ -0,0 +1,27 @@
+SELECT dor.datasource				as datasource,
+		dor.organization			as organization,
+ 		dor.startdate				as startdate,
+ 		dor.enddate					as enddate,
+ 		dor.inferred				as inferred,
+		dor.deletedbyinference		as deletedbyinference,
+		dor.trust					as trust,
+		dor.inferenceprovenance		as inferenceprovenance,
+ 		
+ 		doc.code || '@@@' || doc.name || '@@@' || dos.code || '@@@' || dos.name	as semantics,
+ 		pac.code || '@@@' || pac.name || '@@@' || pas.code || '@@@' || pas.name	as provenanceaction
+ 		
+FROM datasource_organization dor
+
+	left outer join datasources d on (dor.datasource = d.id)
+
+	left outer join class pac on (pac.code = d.provenanceactionclass)
+	left outer join scheme pas on (pas.code = d.provenanceactionscheme)
+
+	left outer join class doc on (doc.code = dor.semanticclass)	
+	left outer join scheme dos on (dos.code = dor.semanticscheme)
+	
+--  These clauses have been commented to allow the import of all the datasource on the index 
+-- WHERE d.datasourceclass != 'entityregistry'
+--	AND d.openairecompatibilityclass IS NOT null
+--	AND d.openairecompatibilityclass != 'UNKNOWN'
+--	AND d.openairecompatibilityclass != 'notCompatible'
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryPersons.sql
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryPersons.sql	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryPersons.sql	(revision 43444)
@@ -0,0 +1,54 @@
+SELECT 	p.id 					        as personid,
+		p.firstname               		as firstname,		
+		p.secondnames                  	as secondnames,
+		CASE WHEN length(p.secondnames) > 0 THEN trim(both from p.secondnames||', '||p.firstname) ELSE p.firstname END as fullname,
+		p.fax			          		as fax,
+		p.email							as email,
+		p.phone		               		as phone,
+		p.dateofcollection				as dateofcollection,
+		p.inferred						as inferred,
+		p.deletedbyinference			as deletedbyinference,
+		p.trust							as trust,
+		p.inferenceprovenance			as inferenceprovenance,
+		dc.id							as collectedfromid,
+		dc.officialname					as collectedfromname,
+		
+		nc.code || '@@@' || nc.name || '@@@' || ns.code || '@@@' || ns.name	as nationality,
+		pac.code || '@@@' || pac.name || '@@@' || pas.code || '@@@' || pas.name	as provenanceaction,
+		
+		array_agg(DISTINCT i.pid || '###' || idc.code || '@@@' || idc.name || '@@@' || ids.code || '@@@' || ids.name)	as pid
+		
+FROM persons p
+	left outer join class nc on (nc.code = p.nationalityclass)	
+	left outer join scheme ns on (ns.code = p.nationalityscheme)
+	
+	left outer join class pac on (pac.code = p.provenanceactionclass)	
+	left outer join scheme pas on (pas.code = p.provenanceactionscheme)	
+	
+	left outer join datasources dc on (dc.id = p.collectedfrom)
+
+	left outer join personpids pp on (pp.person = p.id)
+	left outer join identities i on (i.pid = pp.pid)
+	left outer join class idc on (idc.code = i.issuertypeclass)
+	left outer join scheme ids on (ids.code = i.issuertypescheme)
+	
+	left outer join project_organization po on (po.contactperson = p.id)
+	
+WHERE (p.firstname is not null OR p.secondnames is not null) AND po.resporganization is not null
+	
+GROUP BY
+		p.id,
+		p.firstname,		
+		p.secondnames,
+		p.fax,
+		p.email,
+		p.phone,
+		p.dateofcollection,
+		p.inferred,
+		p.deletedbyinference,
+		p.trust,
+		p.inferenceprovenance,
+		dc.id,
+		dc.officialname,		
+		nc.code, nc.name, ns.code, ns.name,
+		pac.code, pac.name, pas.code, pas.name
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/db/queryClaimsDataset.sql
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/db/queryClaimsDataset.sql	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/db/queryClaimsDataset.sql	(revision 43444)
@@ -0,0 +1 @@
+SELECT regexp_replace(xml, '&(?!amp;)', '&amp;', 'g') as xml, provenance FROM claims WHERE type = 'dmf2actions' and set = 'userclaim_dmf' and provenance = 'user:claim:datacite'
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/db/vocabulary_entries.xquery
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/db/vocabulary_entries.xquery	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/db/vocabulary_entries.xquery	(revision 43444)
@@ -0,0 +1,13 @@
+for $x in collection("/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType")//TERM
+where
+    string-length($x/@english_name) > 0 and string-length($x/@code) > 0
+return
+    <entry>
+        <termEncoding>{$x/@encoding/string()}</termEncoding>
+        <termCode>{$x/@code/string()}</termCode>
+        <termEnglishName>{$x/@english_name/string()}</termEnglishName>
+        <termNativeName>{$x/@native_name/string()}</termNativeName>
+        <vocCode>{$x/../../VOCABULARY_NAME/@code/string()}</vocCode>
+        <vocName>{$x/../../VOCABULARY_NAME/text()}</vocName>
+        <vocDescription>{$x/../../VOCABULARY_DESCRIPTION/text()}</vocDescription>
+    </entry>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/nodes/applicationContext-msro-openaire-nodes.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/nodes/applicationContext-msro-openaire-nodes.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/nodes/applicationContext-msro-openaire-nodes.xml	(revision 43444)
@@ -0,0 +1,352 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:p="http://www.springframework.org/schema/p"
+       xmlns="http://www.springframework.org/schema/beans"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+	<bean id="wfNodeUpdateActionSets"
+	      class="eu.dnetlib.msro.workflows.nodes.actionmanager.UpdateSetsJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodePrepareActionSets"
+	      class="eu.dnetlib.msro.workflows.nodes.actionmanager.PrepareActionSetsJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeCleanActionSetsProfile"
+	      class="eu.dnetlib.msro.workflows.nodes.actionmanager.CleanActionSetsProfileJobNode"
+	      p:xupdate="${dnet.actionmanager.garbage.sets.xupdate}" scope="prototype"/>
+
+	<bean id="wfNodeGarbageSetsHDFS"
+	      class="eu.dnetlib.msro.workflows.nodes.actionmanager.GarbageSetsHDFSJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodePromoteActionsHDFS"
+	      class="eu.dnetlib.msro.workflows.nodes.actionmanager.PromoteActionsHDFSJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodePrepareCopyTable"
+	      class="eu.dnetlib.msro.workflows.nodes.hadoop.PrepareCopyTableJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeDecapsuleClaims"
+	      class="eu.dnetlib.msro.workflows.nodes.claims.DecapsuleClaimsJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeOpenaireMdBuilder"
+	      class="eu.dnetlib.msro.workflows.nodes.OpenaireMdBuilderJobNode"
+	      p:mdBuilderTemplateXslt-ref="openaireMdBuilderTemplateXslt" scope="prototype"/>
+
+	<bean id="openaireMdBuilderTemplateXslt"
+	      class="eu.dnetlib.springutils.stringtemplate.StringTemplateFactory"
+	      p:template="classpath:/eu/dnetlib/msro/workflows/repo-hi/xslt/openaireMdBuilder.xslt.st"
+	      scope="prototype"/>
+
+	<bean id="wfNodeFindComplianceRepositories"
+	      class="eu.dnetlib.msro.workflows.nodes.FindComplianceRepositoriesJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeFindExistingRepositories"
+	      class="eu.dnetlib.msro.workflows.nodes.FindExistingRepositoriesJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeUpdateRepositoryProfiles"
+	      class="eu.dnetlib.msro.workflows.nodes.UpdateRepositoryProfilesJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeUpdateOpenaireDb" class="eu.dnetlib.msro.workflows.nodes.db.UpdateDbJobNode"
+	      scope="prototype" p:db="${dnet.openaire.db.name}"/>
+
+	<bean id="wfNodeFindVocabularyEntries"
+	      class="eu.dnetlib.msro.workflows.nodes.FindVocabularyEntriesJobNode"
+	      scope="prototype"
+	      p:xquery="classpath:/eu/dnetlib/msro/workflows/db/vocabulary_entries.xquery"/>
+
+	<!-- 	<bean id="wfNodeGetHBaseTableDescription" -->
+	<!-- 		class="eu.dnetlib.msro.workflows.nodes.hbase.GetHBaseTableDescriptionJobNode" -->
+	<!-- 		scope="prototype" /> -->
+
+	<bean id="wfNodeImportFiles"
+	      class="eu.dnetlib.msro.workflows.nodes.objectstore.ImportFilesJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeRetreiveInterfaceInfo"
+	      class="eu.dnetlib.msro.workflows.nodes.repohi.RetrieveInterfaceInfoJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeObtainOpenaireDataSourceParams"
+	      class="eu.dnetlib.msro.workflows.nodes.ObtainOpenaireDataSourceParamsJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeSetHdfsFile"
+	      class="eu.dnetlib.msro.workflows.nodes.hadoop.SetHdfsFileJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeFindSearchService"
+	      class="eu.dnetlib.msro.workflows.nodes.index.FindSearchServiceJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeLoadContextsJob"
+	      class="eu.dnetlib.msro.workflows.nodes.LoadContextsJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeLoadEntityLinksJob"
+	      class="eu.dnetlib.msro.workflows.nodes.LoadEntityLinksJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodePrepareIndexJob"
+	      class="eu.dnetlib.msro.workflows.nodes.index.PrepareIndexDataJobNode"
+	      scope="prototype" p:hbaseTable="${hbase.mapred.datatable}"/>
+
+	<bean id="wfNodePrepareStatsParams"
+	      class="eu.dnetlib.msro.workflows.nodes.hadoop.PrepareStatsParamJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeFinalizeIndexFeeding"
+	      class="eu.dnetlib.msro.workflows.nodes.index.FinalizeIndexJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeSwitchIndex"
+	      class="eu.dnetlib.msro.workflows.nodes.index.SwitchIndexJobNode"
+	      scope="prototype"/>
+
+
+	<bean id="wfNodePrepareIisMainParams"
+	      class="eu.dnetlib.msro.workflows.nodes.hadoop.PrepareIISMainParamsJobNode"
+	      scope="prototype" p:xqueryMdStoreService="${dnet.openaire.iis.mdstore.endpoint.xquery}"
+	      p:xqueryDatasetStore="${dnet.openaire.iis.mdstore.dataset.xquery}"
+	      p:xqueryObjectStores="${dnet.openaire.iis.objecstores.xquery}"
+	      p:xqueryObjectStoreService="${dnet.openaire.iis.objecstore.endpoint.xquery}"
+	      p:xqueryIsLookupService="${dnet.openaire.iis.islookup.endpoint.xquery}"/>
+
+	<bean id="wfNodePreparePreprocessParams"
+	      class="eu.dnetlib.msro.workflows.nodes.hadoop.PrepareIISPreprocessingParamsJobNode"
+	      scope="prototype" p:xqueryDatabaseService="${dnet.openaire.iis.dbservice.endpoint.xquery}"
+	      p:xqueryMdStoreService="${dnet.openaire.iis.mdstore.endpoint.xquery}"
+	      p:xqueryObjectStoreService="${dnet.openaire.iis.objecstore.endpoint.xquery}"
+	      p:xqueryDatasetStore="${dnet.openaire.iis.mdstore.dataset.xquery}"
+	      p:xqueryWosMDStore="${dnet.openaire.iis.mdstore.wos.xquery}"
+	      p:xqueryDataciteObjectStore="${dnet.openaire.iis.objecstores.dataset.xquery}"
+	      p:xqueryWosObjectStore="${dnet.openaire.iis.objecstores.wos.xquery}"
+	      p:xqueryIsLookupService="${dnet.openaire.iis.islookup.endpoint.xquery}"/>
+
+	<bean id="wfNodeDownloadFromMetadataOpenaire"
+	      class="eu.dnetlib.msro.workflows.nodes.objectstore.DownloadFromMetadataOpenaire"
+	      scope="prototype"/>
+
+	<bean id="wfNodePatchHostedBy"
+	      class="eu.dnetlib.msro.workflows.nodes.hostedby.PatchHostedByJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeFindHostedBy"
+	      class="eu.dnetlib.msro.workflows.nodes.hostedby.FindHostedByJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeUpsertHostedByApis"
+	      class="eu.dnetlib.msro.workflows.nodes.hostedby.UpsertHostedByApisJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeSplitDatasetRecord"
+	      class="eu.dnetlib.msro.workflows.nodes.datacite.SplitDatasetRecord"
+	      scope="prototype"/>
+
+	<bean id="wfNodeCollectDatasetsByJournal"
+	      class="eu.dnetlib.msro.workflows.nodes.datacite.CollectDatasetsByJournalJobNode"
+	      scope="prototype"/>
+
+	<!-- Stats -->
+	<bean id="wfNodeFindStatsService"
+	      class="eu.dnetlib.msro.workflows.nodes.stats.FindStatsServiceJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodePromoteShadowStats"
+	      class="eu.dnetlib.msro.workflows.nodes.stats.PromoteShadowStatsJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeRefreshShadowCache"
+	      class="eu.dnetlib.msro.workflows.nodes.stats.RefreshShadowCacheJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeValidateShadowStats"
+	      class="eu.dnetlib.msro.workflows.nodes.stats.ValidateShadowStatsJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeBackupStatsCache"
+	      class="eu.dnetlib.msro.workflows.nodes.stats.BackupStatsCacheJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeRestoreStatsCache"
+	      class="eu.dnetlib.msro.workflows.nodes.stats.RestoreStatsCacheJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodePromoteShadowCacheStats"
+	      class="eu.dnetlib.msro.workflows.nodes.stats.PromoteShadowCacheStatsJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeMigrateStatsCache"
+	      class="eu.dnetlib.msro.workflows.nodes.stats.MigrateStatsCacheJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeRunMDStorePlugin"
+	      class="eu.dnetlib.msro.workflows.nodes.RunMDStorePluginJobNode"
+	      scope="prototype"/>
+
+	<!-- Contexts -->
+	<bean id="wfNodeProcessContexts"
+	      class="eu.dnetlib.msro.workflows.nodes.contexts.ProcessContextsJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodePrepareContextDbEntries"
+	      class="eu.dnetlib.msro.workflows.nodes.contexts.PrepareContextDbEntriesJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeSaveContextProfile"
+	      class="eu.dnetlib.msro.workflows.nodes.contexts.SaveContextProfileJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeCheckHDFSCounts"
+	      class="eu.dnetlib.msro.workflows.nodes.hadoop.CheckHDFSCountJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeStoreHBase"
+	      class="eu.dnetlib.msro.workflows.nodes.hadoop.StoreHBaseRecordsJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeDeleteFromHBase"
+	      class="eu.dnetlib.msro.workflows.nodes.hadoop.DeleteHBaseRecordsJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeDeleteHBase"
+	      class="eu.dnetlib.msro.workflows.nodes.hadoop.DeleteHBaseRecordsJobNode"
+	      scope="prototype"/>
+
+	<!-- Data Flow Monitoring -->
+	<bean id="wfNodeTriggerPrepublicMonitoringScenario"
+	      class="eu.dnetlib.msro.workflows.nodes.dfm.TriggerPrepublicMonitoringScenarioJobNode"
+	      scope="prototype"/>
+	<bean id="wfNodeTriggerIndexMonitoringScenario"
+	      class="eu.dnetlib.msro.workflows.nodes.dfm.TriggerIndexMonitoringScenarioJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeFeedMissingClaims" class="eu.dnetlib.msro.workflows.nodes.index.FeedMissingClaimsJobNode"
+	      p:queue-ref="recentPublicationsQueue" p:oafToIndexRecordFactory-ref="apiOafToIndexRecordFactory"
+	      scope="prototype"/>
+
+
+	<bean id="wfNodePrepareHostedbyUpdateSQL"
+	      class="eu.dnetlib.msro.workflows.nodes.hostedby.PrepareHostedbyUpdateSQL"
+	      scope="prototype"/>
+
+	<bean id="wfNodeVREPost"
+	      class="eu.dnetlib.msro.workflows.nodes.vre.VREPostJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeDropHBaseTable"
+	      class="eu.dnetlib.msro.workflows.nodes.hadoop.DropHBaseTableJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeCheckHBaseTable"
+	      class="eu.dnetlib.msro.workflows.nodes.hadoop.ExistHBaseTableJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeDefineHBaseOpenaireSchema"
+	      class="eu.dnetlib.msro.workflows.nodes.hadoop.DefineHBaseOpenaireSchemaJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeCreateHBaseTable"
+	      class="eu.dnetlib.msro.workflows.nodes.hadoop.CreateHBaseTableJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeSubmitHadoopJob"
+	      class="eu.dnetlib.msro.workflows.nodes.hadoop.SubmitHadoopJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeMDStoreBatchExporter"
+	      class="eu.dnetlib.msro.workflows.nodes.hadoop.MDStoreBatchExporterJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeStoreHdfsRecords"
+	      class="eu.dnetlib.msro.workflows.nodes.hadoop.StoreHdfsRecordsJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeReuseHdfsRecords"
+	      class="eu.dnetlib.msro.workflows.nodes.hadoop.ReuseHdfsRecordsJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeLoadMappingProfile"
+	      class="eu.dnetlib.msro.workflows.nodes.LoadMappingProfileJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeBuildSimilarityMeshJob"
+	      class="eu.dnetlib.msro.workflows.nodes.dedup.BuildSimilarityMeshJobNode"
+	      p:similarity-ref="similarityTemplate" scope="prototype"/>
+
+	<bean id="wfNodeFetchRelClasses"
+	      class="eu.dnetlib.msro.workflows.nodes.FetchRelClassesJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeSetDedupConfiguration"
+	      class="eu.dnetlib.msro.workflows.nodes.dedup.DedupConfigurationSetterJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeDedupConfigurationLoaderJob"
+	      class="eu.dnetlib.msro.workflows.nodes.dedup.DedupConfigurationLoaderJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeDedupConfigurationAwareJob"
+	      class="eu.dnetlib.msro.workflows.nodes.dedup.DedupConfigurationAwareJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeCheckEntitySequenceJob"
+	      class="eu.dnetlib.msro.workflows.nodes.dedup.DedupCheckEntitySequenceJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeMinDistSearchHadoopJob"
+	      class="eu.dnetlib.msro.workflows.nodes.dedup.MinDistSearchHadoopJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeCreateHdfsDirJob"
+	      class="eu.dnetlib.msro.workflows.nodes.hadoop.CreateHdfsDirectoryJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeSelectWfPathByDedupEntity"
+	      class="eu.dnetlib.msro.workflows.nodes.dedup.SelectWfPathByDedupEntityJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeQueryUserActionDbJob"
+	      class="eu.dnetlib.msro.workflows.nodes.dedup.QueryUserActionDbJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodePrepareConfiguredActionSet"
+	      class="eu.dnetlib.msro.workflows.nodes.dedup.PrepareConfiguredActionSetJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeDedupSimilarityToActions"
+	      class="eu.dnetlib.msro.workflows.nodes.dedup.DedupSimilarityToActionsJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodePrepareDedupIndexJob"
+	      class="eu.dnetlib.msro.workflows.nodes.dedup.PrepareDedupIndexJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeFinalizeDedupIndexFeeding"
+	      class="eu.dnetlib.msro.workflows.nodes.dedup.FinalizeDedupIndexJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeIndexDsUpdateJob"
+	      class="eu.dnetlib.msro.workflows.nodes.IndexDSUpdateJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeDeleteHdfsPathJob"
+	      class="eu.dnetlib.msro.workflows.nodes.hadoop.DeleteHdfsPathJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeDuplicateScanJob"
+	      class="eu.dnetlib.msro.workflows.nodes.dedup.DedupDuplicateScanJobNode"
+	      scope="prototype"/>
+
+	<bean id="wfNodeFindIndex"
+	      class="eu.dnetlib.msro.workflows.nodes.provision.FindIndexJobNode"
+	      scope="prototype"/>
+
+
+</beans>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/openaire/api/findIndexDsInfo.xquery
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/openaire/api/findIndexDsInfo.xquery	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/openaire/api/findIndexDsInfo.xquery	(revision 43444)
@@ -0,0 +1,9 @@
+let $format := collection('/db/DRIVER/ServiceResources/SearchServiceResourceType')//SERVICE_PROPERTIES[./PROPERTY[@key = 'infrastructure' and @value = 'public']]/PROPERTY[@key = "mdformat"]/@value/string()
+
+for $x in collection('/db/DRIVER/IndexDSResources/IndexDSResourceType')
+where
+    $x//METADATA_FORMAT = $format and
+            $x//METADATA_FORMAT_INTERPRETATION = 'openaire' and
+            $x//METADATA_FORMAT_LAYOUT = 'index'
+return
+    concat($x//RESOURCE_IDENTIFIER/@value/string(), ' @@@ ', $format, ' @@@ ', $format, '-index-openaire')
Index: modules/dnet-wds/trunk/src/main/java/eu/dnetlib/wds/collector/plugins/CMRDatasourcePlugin.java
===================================================================
--- modules/dnet-wds/trunk/src/main/java/eu/dnetlib/wds/collector/plugins/CMRDatasourcePlugin.java	(revision 43443)
+++ modules/dnet-wds/trunk/src/main/java/eu/dnetlib/wds/collector/plugins/CMRDatasourcePlugin.java	(revision 43444)
@@ -11,7 +11,9 @@
 import eu.dnetlib.rmi.data.CollectorServiceException;
 import eu.dnetlib.rmi.data.InterfaceDescriptor;
 import eu.dnetlib.rmi.data.plugin.AbstractCollectorPlugin;
+import org.antlr.stringtemplate.StringTemplate;
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
 
 /**
  * Created by sandro on 7/1/16.
@@ -20,10 +22,21 @@
 
 	private static final String BASE_URL = "https://cmr.earthdata.nasa.gov/search/keywords/data_centers?pretty=true";
 
+	private StringTemplate xmlTemplate;
+
+
+
 	@Override
 	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)
 			throws CollectorServiceException {
+
+		return generateListOfDatasource();
+
+	}
+
+	public List<String> generateListOfDatasource() throws CollectorServiceException {
 		try {
+
 			URL myUrl = new URL(BASE_URL);
 
 			JsonElement element = new JsonParser().parse(IOUtils.toString(myUrl.openStream()));
@@ -40,6 +53,7 @@
 
 				if (short_name != null) {
 					final JsonArray level_1 = short_name.getAsJsonArray();
+
 					for (JsonElement item : level_1) {
 						final JsonObject currentItem = item.getAsJsonObject();
 						final String id = currentItem.get("value").getAsString();
@@ -58,13 +72,32 @@
 							final JsonElement urlObject = infoObject.get("url");
 
 							String url = "";
+							String uuid = "";
 							if (urlObject != null) {
 								if (urlObject.isJsonArray()) {
 									url = urlObject.getAsJsonArray().get(0).getAsJsonObject().get("value").getAsString();
-								} else url = urlObject.getAsJsonObject().get("value").getAsString();
+									uuid = urlObject.getAsJsonArray().get(0).getAsJsonObject().get("uuid").getAsString();
+								} else {
+									url = urlObject.getAsJsonObject().get("value").getAsString();
+									uuid = urlObject.getAsJsonObject().get("uuid").getAsString();
+								}
 							}
 
-							results.add(id + "<---->" + name + " <----------------->" + url);
+							CMRDatasource currentDatasource = new CMRDatasource();
+
+							try {
+								currentDatasource.setId(id);
+								currentDatasource.setName(name);
+								currentDatasource.setUrl(url);
+								currentDatasource.setUuid(uuid);
+								if (!StringUtils.isBlank(id) && !StringUtils.isBlank(name) && !StringUtils.isBlank(url)) {
+									xmlTemplate.removeAttribute("datasource");
+									xmlTemplate.setAttribute("datasource", currentDatasource);
+									results.add(xmlTemplate.toString());
+								}
+							} catch (Throwable e1) {
+								System.out.println("ERRORE");
+							}
 						}
 					}
 				}
@@ -73,7 +106,14 @@
 		} catch (Throwable e) {
 			throw new CollectorServiceException("Error on iterating CMR data centers", e);
 		}
+	}
 
+	public StringTemplate getXmlTemplate() {
+		return xmlTemplate;
 	}
 
+	public void setXmlTemplate(final StringTemplate xmlTemplate) {
+		this.xmlTemplate = xmlTemplate;
+	}
+
 }
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/dedup/similarityTemplate.st
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/dedup/similarityTemplate.st	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/dedup/similarityTemplate.st	(revision 43444)
@@ -0,0 +1,5 @@
+<SIMILARITY>
+	<SOURCE id="$source$"/>
+	<TARGET id="$target$"/>
+	<TYPE name="$type$"/>
+</SIMILARITY>
\ No newline at end of file
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryProjectOrganization.sql
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryProjectOrganization.sql	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryProjectOrganization.sql	(revision 43444)
@@ -0,0 +1,22 @@
+SELECT	po.project						as project,
+ 		po.resporganization				as resporganization,
+		po.participantnumber			as participantnumber, 
+ 		po.startdate					as startdate,
+ 		po.enddate						as enddate,
+ 		po.inferred						as inferred, 
+ 		po.deletedbyinference			as deletedbyinference, 
+ 		po.trust						as trust, 
+ 		po.inferenceprovenance			as inferenceprovenance,
+ 		
+ 		poc.code || '@@@' || poc.name || '@@@' || pos.code || '@@@' || pos.name	as semantics,
+ 		pac.code || '@@@' || pac.name || '@@@' || pas.code || '@@@' || pas.name	as provenanceaction
+
+FROM project_organization po
+
+	left outer join projects p on (po.project = p.id) 
+
+	left outer join class pac on (pac.code = p.provenanceactionclass)
+	left outer join scheme pas on (pas.code = p.provenanceactionscheme)
+
+	left outer join class poc on (poc.code = po.semanticclass)	
+	left outer join scheme pos on (pos.code = po.semanticscheme)
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryDatasourcesForSize.sql
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryDatasourcesForSize.sql	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryDatasourcesForSize.sql	(revision 43444)
@@ -0,0 +1 @@
+SELECT count(*) FROM datasources;
\ No newline at end of file
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryProjectContactperson.sql
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryProjectContactperson.sql	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryProjectContactperson.sql	(revision 43444)
@@ -0,0 +1,15 @@
+SELECT distinct 
+		po.project, 
+		po.contactperson,
+		po.startdate,
+		po.enddate,
+		po.inferred,
+		po.deletedbyinference,
+		po.trust,
+		po.inferenceprovenance,
+		'contactperson@@@contactperson@@@dnet:project_organization_relations@@@dnet:project_organization_relations'	as semantics
+		
+FROM project_organization po
+LEFT OUTER JOIN persons p on (p.id = po.contactperson)
+
+WHERE po.contactperson is not null AND (p.firstname is not null OR p.secondnames is not null)
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/db/decapsule_claims_xml.xsl
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/db/decapsule_claims_xml.xsl	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/db/decapsule_claims_xml.xsl	(revision 43444)
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+
+	<xsl:output omit-xml-declaration="yes" indent="yes"/>
+
+	<xsl:template match="/">
+		<ROW>
+			<xsl:apply-templates/>
+		</ROW>
+	</xsl:template>
+
+	<xsl:template match="FIELD[@name = 'xml']">
+		<xsl:value-of select="." disable-output-escaping="yes"/>
+	</xsl:template>
+
+	<xsl:template match="FIELD[@name != 'xml']">
+		<xsl:copy-of select="."/>
+	</xsl:template>
+
+</xsl:stylesheet>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/db/queryACM.sql
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/db/queryACM.sql	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/db/queryACM.sql	(revision 43444)
@@ -0,0 +1 @@
+SELECT regexp_replace(xml, '&(?!amp;)', '&amp;', 'g') as xml, provenance FROM claims WHERE type = 'dmf2actions' and set = 'acm-referenced-projects'
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/nodes/applicationContext-msro-openaire.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/nodes/applicationContext-msro-openaire.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/nodes/applicationContext-msro-openaire.xml	(revision 43444)
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:p="http://www.springframework.org/schema/p"
+       xmlns="http://www.springframework.org/schema/beans"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+	<bean id="contextDsLoader" class="eu.dnetlib.msro.workflows.nodes.index.ContextLoader"
+	      p:xquery="${dnet.openaire.context.xquery}"/>
+
+	<bean id="entityGrouperConfLoader" class="eu.dnetlib.msro.workflows.nodes.index.EntityGrouperConfigurationLoader"
+	      p:xquery="${dnet.openaire.entitygruper.xquery}"/>
+
+	<bean id="similarityTemplate"
+	      class="eu.dnetlib.springutils.stringtemplate.StringTemplateFactory"
+	      p:template="classpath:/eu/dnetlib/msro/workflows/dedup/similarityTemplate.st"
+	      scope="prototype"/>
+
+	<bean id="dedupConfigurationOrchestrationLoader"
+	      class="eu.dnetlib.msro.workflows.nodes.dedup.utils.DedupConfigurationOrchestrationLoader"/>
+
+
+</beans>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/openaire/api/findSolrIndexUrl.xquery
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/openaire/api/findSolrIndexUrl.xquery	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/openaire/api/findSolrIndexUrl.xquery	(revision 43444)
@@ -0,0 +1 @@
+collection("/db/DRIVER/ServiceResources/IndexServiceResourceType")//PROTOCOL[@name = "solr" or @name = "SOLR"]/@address/string()
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/dedup/querySimilarities.sql
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/dedup/querySimilarities.sql	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/dedup/querySimilarities.sql	(revision 43444)
@@ -0,0 +1,5 @@
+SELECT * FROM (
+	SELECT g.id, date, entitytype, array_agg(sg.objidentifier) as group 
+	FROM groups g LEFT OUTER join similarity_groups sg ON (g.id = sg.groupid) 
+	GROUP BY g.id, date, entitytype) AS T 
+WHERE array_length(T.group, 1) > 1
\ No newline at end of file
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryClaimsRel.sql
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryClaimsRel.sql	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryClaimsRel.sql	(revision 43444)
@@ -0,0 +1 @@
+SELECT xml, provenance FROM claims WHERE type = 'rels2actions' and set = 'userclaim_result_project'	or set = 'userclaim_result_result'
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryDatasources.sql
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryDatasources.sql	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryDatasources.sql	(revision 43444)
@@ -0,0 +1,161 @@
+SELECT d.id AS datasourceid,
+d.officialname AS officialname,
+d.englishname AS englishname,
+
+CASE
+WHEN (array_agg( DISTINCT tcc.compliance :: TEXT ) @> ARRAY ['driver', 'openaire2.0'])
+THEN
+				'driver-openaire2.0@@@OpenAIRE 2.0+ (DRIVER OA, EC funding)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+WHEN (array_agg( DISTINCT tcc.compliance :: TEXT ) @> ARRAY ['driver'])
+THEN
+				'driver@@@OpenAIRE Basic (DRIVER OA)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+WHEN (array_agg( DISTINCT tcc.compliance :: TEXT ) @> ARRAY ['openaire2.0'])
+THEN
+				'openaire2.0@@@OpenAIRE 2.0 (EC funding)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+WHEN (array_agg( DISTINCT tcc.compliance :: TEXT ) @> ARRAY ['openaire3.0'])
+THEN
+				'openaire3.0@@@OpenAIRE 3.0 (OA, funding)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+WHEN (array_agg( DISTINCT tcc.compliance :: TEXT ) @> ARRAY ['openaire2.0_data'])
+THEN
+'openaire2.0_data@@@OpenAIRE Data (funded, referenced datasets)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+WHEN (array_agg( DISTINCT tcc.compliance :: TEXT ) @> ARRAY ['native'])
+THEN
+				'native@@@proprietary@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+WHEN (array_agg( DISTINCT tcc.compliance :: TEXT ) @> ARRAY ['hostedBy'])
+THEN
+	'hostedBy@@@collected from a compatible aggregator@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+WHEN (array_agg( DISTINCT tcc.compliance :: TEXT ) @> ARRAY ['notCompatible'])
+THEN
+				'notCompatible@@@under validation@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+ELSE
+				'UNKNOWN@@@not available@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+END AS openairecompatibility,
+
+d.websiteurl AS websiteurl,
+d.logourl AS logourl,
+d.contactemail AS contactemail,
+array_agg( DISTINCT CASE
+			WHEN apc.param='baseUrl' and api.protocolclass='oai' THEN
+				CASE WHEN COALESCE(apc.edited, '') = ''  THEN apc.original ELSE apc.edited END
+			ELSE
+				NULL
+END ) AS accessinfopackage, -- we need baseUrl param only -when available-, but we can not use WHERE clause on apc.param, otherwise we exclude datasources without a baseUrl param and the returned rows are less than we expect.
+		d.latitude					as latitude,
+		d.longitude					as longitude,
+		d.namespaceprefix			as namespaceprefix,
+		d.od_numberofitems			as odnumberofitems,          	
+		d.od_numberofitemsdate		as odnumberofitemsdate,      		
+		array_agg(DISTINCT s.name || '###' || sc.code || '@@@' || sc.name || '@@@' || ss.code || '@@@' || ss.name) as subjects,
+		d.description				as description,
+		d.od_policies				as odpolicies,
+		ARRAY(select trim(s) from unnest(string_to_array(d.od_languages, ',')) as s)  as odlanguages,
+		ARRAY(select trim(s) from unnest(string_to_array(d.od_contenttypes, '-')) as s) as odcontenttypes,
+		d.inferred					as inferred,
+		d.deletedbyinference		as deletedbyinference,
+		d.trust						as trust,
+		d.inferenceprovenance		as inferenceprovenance,
+		d.dateofcollection			as dateofcollection,
+		d.dateofvalidation			as dateofvalidation,
+		
+		-- re3data fields
+		d.releasestartdate			as releasestartdate,
+		d.releaseenddate			as releaseenddate,
+		d.missionstatementurl		as missionstatementurl,
+		d.dataprovider				as dataprovider,
+		d.serviceprovider			as serviceprovider,
+		d.databaseaccesstype		as databaseaccesstype,
+		d.datauploadtype			as datauploadtype,
+		d.databaseaccessrestriction	as databaseaccessrestriction,
+		d.datauploadrestriction		as datauploadrestriction,
+		d.versioning				as versioning,
+		d.citationguidelineurl		as citationguidelineurl,
+		d.qualitymanagementkind		as qualitymanagementkind,
+		d.pidsystems				as pidsystems,
+		d.certificates				as certificates,
+		array_agg(DISTINCT p.name || '&&&' || p.url) as policies,
+		-- end of re3data fields
+		
+		dc.id					as collectedfromid,
+		dc.officialname			as collectedfromname,
+		
+		tc.code || '@@@' || tc.name || '@@@' || ts.code || '@@@' || ts.name	as datasourcetype,
+		pac.code || '@@@' || pac.name || '@@@' || pas.code || '@@@' || pas.name	as provenanceaction,
+
+		array_agg(DISTINCT i.pid || '###' || idc.code || '@@@' || idc.name || '@@@' || ids.code || '@@@' || ids.name) as pid
+		
+
+FROM datasources d
+	left outer join class tc on (tc.code = d.datasourceclass)	
+	left outer join scheme ts on (ts.code = d.datasourcescheme)
+	
+	left outer join class pac on (pac.code = d.provenanceactionclass)	
+	left outer join scheme pas on (pas.code = d.provenanceactionscheme)		
+
+	left outer join datasourcepids dps on (dps.datasource = d.id)
+	left outer join identities i on (i.pid = dps.pid)
+	
+	left outer join class idc on (idc.code = i.issuertypeclass)
+	left outer join scheme ids on (ids.code = i.issuertypescheme)
+	
+	left outer join datasources dc on (dc.id = d.collectedfrom)
+	left outer join api on (api.datasource = d.id)
+	left outer join apicollections apc on (apc.api = api.id)	
+	
+	left outer join temp_compliances tcc on (tcc.api = api.id)	
+	
+	left outer join datasource_subject ds on (ds.datasource = d.id)
+	left outer join subjects s on (s.id = ds.subject) 
+	
+	left outer join class sc on (sc.code = s.semanticclass)
+	left outer join scheme ss on (ss.code = s.semanticscheme)
+	
+	left outer join datasource_policy dp on (dp.datasource = d.id)
+	left outer join policies p on (p.id = dp.policy)
+	
+--  These clauses have been commented to allow the import of all the datasource on the index 
+-- WHERE d.datasourceclass != 'entityregistry'
+--	AND d.openairecompatibilityclass IS NOT null
+--	AND d.openairecompatibilityclass != 'UNKNOWN'
+--	AND d.openairecompatibilityclass != 'notCompatible'
+	
+GROUP BY 
+		d.id,
+		d.officialname, 
+		d.englishname,
+		d.websiteurl,
+		d.logourl,
+		d.contactemail,
+		d.namespaceprefix,
+		d.description,
+		d.od_numberofitems,          	
+		d.od_numberofitemsdate,      		
+		d.od_policies,
+		d.od_languages,         		
+		d.od_contenttypes,
+		d.latitude,
+		d.longitude,
+		d.inferred,
+		d.deletedbyinference,
+		d.trust,
+		d.inferenceprovenance,
+		d.dateofcollection,
+		d.dateofvalidation,
+		dc.id,
+		d.releasestartdate,
+		d.releaseenddate,
+		d.missionstatementurl,
+		d.dataprovider,
+		d.serviceprovider,
+		d.databaseaccesstype,
+		d.datauploadtype,
+		d.databaseaccessrestriction,
+		d.datauploadrestriction,
+		d.versioning,
+		d.citationguidelineurl,
+		d.qualitymanagementkind,
+		d.pidsystems,
+		d.certificates,
+		dc.officialname,
+		tc.code, tc.name, ts.code, ts.name, 
+		pac.code, pac.name, pas.code, pas.name,
+		sc.code, sc.name, ss.code, ss.name
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryOrganizations.sql
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryOrganizations.sql	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/hbase/queryOrganizations.sql	(revision 43444)
@@ -0,0 +1,69 @@
+SELECT 	 o.id										as organizationid, 
+		 o.legalshortname							as legalshortname,
+		 o.legalname                                as legalname,
+		 o.websiteurl                               as websiteurl,
+		 o.logourl                                  as logourl,
+		 o.ec_legalbody                             as eclegalbody,
+		 o.ec_legalperson                           as eclegalperson,
+		 o.ec_nonprofit                             as ecnonprofit,
+		 o.ec_researchorganization                  as ecresearchorganization,
+		 o.ec_highereducation                       as echighereducation,
+		 o.ec_internationalorganizationeurinterests as ecinternationalorganizationeurinterests,
+		 o.ec_internationalorganization             as ecinternationalorganization,
+		 o.ec_enterprise                            as ecenterprise,
+		 o.ec_smevalidated                          as ecsmevalidated,
+		 o.ec_nutscode                              as ecnutscode,
+		 o.dateofcollection							as dateofcollection,
+		 o.inferred                                 as inferred,
+		 o.deletedbyinference                       as deletedbyinference,
+		 o.trust                                    as trust,
+		 o.inferenceprovenance                      as inferenceprovenance,
+		 dc.id			                            as collectedfromid,
+		 dc.officialname                            as collectedfromname,
+		 
+		 cc.code || '@@@' || cc.name || '@@@' || cs.code || '@@@' || cs.name	as country,
+		 pac.code || '@@@' || pac.name || '@@@' || pas.code || '@@@' || pas.name	as provenanceaction,
+		 
+		 array_agg(DISTINCT i.pid || '###' || idc.code || '@@@' || idc.name || '@@@' || ids.code || '@@@' || ids.name)	as pid
+
+FROM organizations o
+	left outer join class cc on (cc.code = o.countryclass)	
+	left outer join scheme cs on (cs.code = o.countryscheme)
+	
+	left outer join class pac on (pac.code = o.provenanceactionclass)	
+	left outer join scheme pas on (pas.code = o.provenanceactionscheme)		
+	
+	left outer join organizationpids op on (op.organization = o.id)
+	left outer join identities i on (i.pid = op.pid)
+	
+	left outer join datasources dc on (dc.id = o.collectedfrom)
+	
+	left outer join class idc on (idc.code = i.issuertypeclass)
+	left outer join scheme ids on (ids.code = i.issuertypescheme)	
+ 
+GROUP BY
+		 o.id, 
+		 o.legalshortname,
+		 o.legalname,
+		 o.websiteurl,
+		 o.logourl,
+		 o.ec_legalbody,
+		 o.ec_legalperson,
+		 o.ec_nonprofit,
+		 o.ec_researchorganization,
+		 o.ec_highereducation,
+		 o.ec_internationalorganizationeurinterests,
+		 o.ec_internationalorganization,
+		 o.ec_enterprise,
+		 o.ec_smevalidated,
+		 o.ec_nutscode,
+		 o.dateofcollection,
+		 o.inferred,
+		 o.deletedbyinference,
+		 o.trust,
+		 o.inferenceprovenance,
+		 dc.id, 
+		 dc.officialname, 
+		 cc.code, cc.name, cs.code, cs.name,
+		 pac.code, pac.name, pas.code, pas.name
+		
\ No newline at end of file
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/repo-hi/xslt/openaireMdBuilder.xslt.st
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/repo-hi/xslt/openaireMdBuilder.xslt.st	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/repo-hi/xslt/openaireMdBuilder.xslt.st	(revision 43444)
@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xmlns:datetime="http://exslt.org/dates-and-times" xmlns:exslt="http://exslt.org/common"
+    xmlns:oai="http://www.openarchives.org/OAI/2.0/"
+    xmlns:dnet="eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions"
+    xmlns:oaf="http://namespace.openaire.eu/oaf"
+    exclude-result-prefixes="xsl datetime exslt dnet"
+    xmlns:dri="http://www.driver-repository.eu/namespace/dri">
+    
+    <xsl:variable name="status" select="/*[local-name() = 'record']/*[local-name() = 'header']/@status" />
+    
+    <xsl:template match="/">
+        <oai:record xmlns="http://namespace.openaire.eu/"
+            xmlns:dc="http://purl.org/dc/elements/1.1/"
+            xmlns:dr="http://www.driver-repository.eu/namespace/dr"
+            xmlns:prov="http://www.openarchives.org/OAI/2.0/provenance"
+            xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+            
+            <xsl:variable name="datestamp" select="datetime:dateTime()" />
+            
+            <oai:header>
+                <xsl:if test="\$status">
+                    <xsl:attribute name="status" >
+                        <xsl:value-of select="\$status"/>
+                    </xsl:attribute>
+                </xsl:if>
+                <xsl:if test="not(.//dri:objIdentifier)">
+                    <dri:objIdentifier><xsl:value-of select="concat('$namespacePrefix$::', dnet:md5($xpath$))"/></dri:objIdentifier>
+                </xsl:if>
+                <xsl:if test="not(.//dri:recordIdentifier)">
+                    <dri:recordIdentifier><xsl:value-of select="$xpath$"/></dri:recordIdentifier>
+                </xsl:if>
+                <xsl:if test="not(.//dri:dateOfCollection)">
+                    <dri:dateOfCollection><xsl:value-of select="\$datestamp"/></dri:dateOfCollection>
+                </xsl:if>
+                <xsl:if test="not(.//dri:repositoryId)">
+                    <dri:repositoryId>$datasourceId$</dri:repositoryId>
+                </xsl:if>
+                <xsl:if test="not(.//oaf:datasourceprefix)">
+                    <oaf:datasourceprefix>$namespacePrefix$</oaf:datasourceprefix>
+                </xsl:if>
+                
+                <xsl:for-each select="/*[local-name() = 'record']/*[local-name() = 'header']/*">
+                    <xsl:copy-of select="."/>
+                </xsl:for-each>
+            </oai:header>
+            <xsl:choose>
+                <xsl:when test="count(/*[local-name() = 'record']/*[local-name() = 'metadata']) &gt; 0">
+                    <xsl:copy-of select="/*[local-name() = 'record']/*[local-name() = 'metadata']"/>
+                </xsl:when>
+                <xsl:when test="\$status = 'deleted'" />
+                <xsl:otherwise>
+                    <oai:metadata>
+                        <xsl:copy-of select="."/>
+                    </oai:metadata>
+                </xsl:otherwise>
+            </xsl:choose>
+            <xsl:if test="not(\$status) or \$status != 'deleted'">
+                <about>
+	                <provenance xmlns="http://www.openarchives.org/OAI/2.0/provenance" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/provenance http://www.openarchives.org/OAI/2.0/provenance.xsd">
+	  					<originDescription harvestDate="{\$datestamp}" altered="true">
+		    				<baseURL>$baseurl$</baseURL>
+		    				<identifier><xsl:value-of select="//*[local-name()='header']/*[local-name()='identifier']"/></identifier>
+		    				<datestamp><xsl:value-of select="//*[local-name()='header']/*[local-name()='datestamp']"/></datestamp>
+		    				<metadataNamespace>$metadatanamespace$</metadataNamespace>
+		    				<xsl:copy-of select="//*[local-name()='provenance']/*[local-name() = 'originDescription']"/>
+	  					</originDescription>
+					</provenance>
+                    <oaf:datainfo>
+                        <oaf:inferred>$inferred$</oaf:inferred>
+                        <oaf:deletedbyinference>$deletedbyinference$</oaf:deletedbyinference>
+                        <oaf:trust>$trust$</oaf:trust>
+                        <oaf:inferenceprovenance>$inferenceprovenance$</oaf:inferenceprovenance>
+                        <oaf:provenanceaction schemename="dnet:provenanceActions" schemeid="dnet:provenanceActions" classname="$provenanceactionclassname$" classid="$provenanceactionclassid$"/>
+                    </oaf:datainfo>				
+                </about>
+            </xsl:if>
+        </oai:record>
+    </xsl:template>
+    
+</xsl:stylesheet>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/db/vocEntry_2_db.xsl
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/db/vocEntry_2_db.xsl	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/db/vocEntry_2_db.xsl	(revision 43444)
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+
+	<xsl:template match="/">
+		<ROWS>
+			<ROW table="class">
+				<FIELD name="_dnet_resource_identifier_">
+					<xsl:value-of select="//termCode"/>
+				</FIELD>
+				<FIELD name="code">
+					<xsl:value-of select="//termCode"/>
+				</FIELD>
+				<FIELD name="name">
+					<xsl:value-of select="//termEnglishName"/>
+				</FIELD>
+				<FIELD name="optional1">
+					<xsl:value-of select="//termEncoding"/>
+				</FIELD>
+				<FIELD name="optional2">
+					<xsl:value-of select="//termNativeName"/>
+				</FIELD>
+			</ROW>
+			<ROW table="scheme">
+				<FIELD name="_dnet_resource_identifier_">
+					<xsl:value-of select="//vocCode"/>
+				</FIELD>
+				<FIELD name="code">
+					<xsl:value-of select="//vocCode"/>
+				</FIELD>
+				<FIELD name="name">
+					<xsl:value-of select="//vocName"/>
+				</FIELD>
+				<FIELD name="description">
+					<xsl:value-of select="//vocDescription"/>
+				</FIELD>
+			</ROW>
+			<ROW table="class_scheme">
+				<FIELD name="_dnet_resource_identifier_">
+					<xsl:value-of select="concat(//termCode,'@@',//vocCode)"/>
+				</FIELD>
+				<FIELD name="class">
+					<xsl:value-of select="//termCode"/>
+				</FIELD>
+				<FIELD name="scheme">
+					<xsl:value-of select="//vocCode"/>
+				</FIELD>
+			</ROW>
+		</ROWS>
+	</xsl:template>
+</xsl:stylesheet>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/nodes/applicationContext-msro-openaire.properties
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/nodes/applicationContext-msro-openaire.properties	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/msro/workflows/nodes/applicationContext-msro-openaire.properties	(revision 43444)
@@ -0,0 +1,24 @@
+dnet.openaire.db.name=dnet_openaireplus
+hbase.mapred.datatable=db_openaireplus_test
+#services.data.hadoop.hdfs.mdrecords.tmp.directory = hdfs://nmis-hadoop-cluster/tmp
+dnet.openaire.entitygruper.xquery=/RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'EntityGrouperConfigurationDSResourceType']//CONFIGURATION/GROUP/text()
+dnet.openaire.context.xquery=for $x in //RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='ContextDSResourceType']//*[name()='context' or name()='category' or name()='concept'] return <entry id="{$x/@id}" label="{$x/@label|$x/@name}" name="{$x/name()}" type="{$x/@type}"/>
+dnet.openaire.context.profiles.xquery=for $x in //RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='ContextDSResourceType' and .//RESOURCE_KIND/@value='ContextDSResources'] return $x
+dnet.actionmanager.garbage.sets.xupdate=for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'ActionManagerSetDSResourceType'] let $latest:=$x//LATEST return update replace $x//RAW_SETS with <RAW_SETS>{$latest}</RAW_SETS>
+dnet.openaire.iis.objecstores.xquery=for $x in /RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value = 'ObjectStoreDSResourceType'] return string($x/HEADER/RESOURCE_IDENTIFIER/@value)
+dnet.openaire.iis.objecstore.endpoint.xquery=/RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value = 'ObjectStoreServiceResourceType']/HEADER/PROTOCOLS/PROTOCOL[./@name = "SOAP"]/@address/string()
+dnet.openaire.iis.islookup.endpoint.xquery=/RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value = 'IS_LookUpServiceResourceType']/HEADER/PROTOCOLS/PROTOCOL[./@name = "SOAP"]/@address/string()
+dnet.openaire.iis.mdstore.endpoint.xquery=/RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value = 'MDStoreServiceResourceType']/HEADER/PROTOCOLS/PROTOCOL[./@name = "SOAP"]/@address/string()
+dnet.openaire.iis.dbservice.endpoint.xquery=/RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value = 'DatabaseServiceResourceType']/HEADER/PROTOCOLS/PROTOCOL[./@name = "SOAP"]/@address/string()
+dnet.openaire.iis.mdstore.wos.xquery=for $x in /RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value = 'MDStoreDSResourceType' and .//METADATA_FORMAT = 'OAF' and .//METADATA_FORMAT_LAYOUT = 'store' and .//METADATA_FORMAT_INTERPRETATION = 'cleaned_noingestion'] return string($x/HEADER/RESOURCE_IDENTIFIER/@value)
+dnet.openaire.iis.mdstore.dataset.xquery=for $x in /RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value = 'MDStoreDSResourceType' and .//METADATA_FORMAT = 'ODF' and .//METADATA_FORMAT_LAYOUT = 'store' and (.//METADATA_FORMAT_INTERPRETATION = 'cleaned_noingestion' or .//METADATA_FORMAT_INTERPRETATION = 'cleaned')] return string($x/HEADER/RESOURCE_IDENTIFIER/@value)
+dnet.openaire.iis.objecstores.dataset.xquery=for $x in /RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value = 'ObjectStoreDSResourceType' and .//OBJECTSTORE_INTERPRETATION='pubrepository record'] return string($x/HEADER/RESOURCE_IDENTIFIER/@value)
+dnet.openaire.iis.objecstores.wos.xquery=for $x in /RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value = 'ObjectStoreDSResourceType' and .//OBJECTSTORE_INTERPRETATION='wos record'] return string($x/HEADER/RESOURCE_IDENTIFIER/@value)
+dnet.openaire.model.relclasses.xquery=distinct-values(for $x in /RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value='OntologyDSResourceType']//TERM return concat($x/@code/string(), ":", $x//RELATION[./@type = 'inverseOf']/@code/string()))
+dnet.openaire.service.search.lookup.xquery=/RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='SearchServiceResourceType' and .//RESOURCE_KIND/@value='ServiceResources' and .//SERVICE_PROPERTIES/PROPERTY[./@key="infrastructure"]/@value="public"]/HEADER/RESOURCE_IDENTIFIER/@value/string()
+dnet.openaire.service.stats.lookup.xquery=/RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='StatsManagerServiceResourceType' and .//RESOURCE_KIND/@value='ServiceResources']/HEADER/RESOURCE_IDENTIFIER/@value/string()
+dnet.openaire.similaritygroups.index.format=DEDUP
+dnet.openaire.similaritygroups.indexrecord2html.xsl=classpath:/eu/dnetlib/functionality/modular/ui/xslt/indexRecordsToHtml.xsl
+dnet.openaire.similaritygroups.indexrecord2htmldetails.xsl=classpath:/eu/dnetlib/functionality/modular/ui/xslt/indexRecordsToHtmlDetails.xsl
+oaf.schema.location=http://www.openaire.eu/schema/0.2/oaf-0.2.xsd
+dnet.dedup.db.name=dnet_dedup
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/openaire/api/indexRecord.xml.vm
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/openaire/api/indexRecord.xml.vm	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/openaire/api/indexRecord.xml.vm	(revision 43444)
@@ -0,0 +1,117 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+#macro(context $ctx)
+        <$!ctx.elem id="$!esc.xml($!ctx.id)" label="$!esc.xml($contexts.get($!ctx.id))" #if($!ctx.isRoot()) type="community"#end>
+	#foreach($child in $!ctx.children)#context($child)#end
+        </$!ctx.elem>
+#end
+-->
+
+<record xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:oaf="http://namespace.openaire.eu/oaf">
+	<result>
+		<header>
+			<dri:objIdentifier>$!esc.xml($!objIdentifier)</dri:objIdentifier>
+			<dri:dateOfCollection>$!esc.xml($!dateOfCollection)</dri:dateOfCollection>
+			<dri:status>under curation</dri:status>
+			<counters/>
+		</header>
+		<metadata>
+			<oaf:entity schemaLocation="http://namespace.openaire.eu/oaf $!oafSchemaLocation">
+				<oaf:result>
+					<title classid="main title" classname="main title" schemeid="dnet:dataCite_title" schemename="dnet:dataCite_title">
+						$!esc.xml($!pub.title)
+					</title>
+					<dateofacceptance/>
+					<resulttype classid="$!esc.xml($!pub.type)" classname="$!esc.xml($!resultTypes.get($!pub.type))" schemeid="dnet:result_typologies" schemename="dnet:result_typologies"/>
+					<language classid="$!esc.xml($!pub.language)" classname="$!esc.xml($!languages.get($!pub.language))" schemeid="dnet:languages"
+					          schemename="dnet:languages"/>
+					<description>
+						$!esc.xml($!pub.description)
+					</description>
+					<country classid="" classname="" schemeid="" schemename=""/>
+					<subject classid="" classname="" schemeid="" schemename=""/>
+					<relevantdate classid="" classname="" schemeid="" schemename=""/>
+					<publisher>$!esc.xml($!pub.publisher)</publisher>
+					<embargoenddate>$!esc.xml($!pub.embargoEndDate)</embargoenddate>
+					<source/>
+					<fulltext/>
+					<format/>
+					<storagedate/>
+					<resourcetype classid="" classname="" schemeid="" schemename=""/>
+					<device/>
+					<size/>
+					<version/>
+					<lastmetadataupdate/>
+					<metadataversionnumber/>
+					<originalId>$!esc.xml($!pub.originalId)</originalId>
+					<collectedfrom name="$!esc.xml($!collectedFrom.name)" id="$!esc.xml($!collectedFrom.calculateOpenaireId())"/>
+
+					#foreach($pid in $!pub.pids)
+						<pid classid="$!esc.xml($!pid.type)" classname="$!esc.xml($!pidTypes.get($!pid.type))" schemeid="dnet:pid_types"
+						     schemename="dnet:pid_types">$!esc.xml($pid.value)</pid>
+					#end
+
+					<bestlicense classid="$!esc.xml($!pub.licenseCode)" classname="$!esc.xml($!licenses.get($!pub.licenseCode))"
+					             schemeid="dnet:access_modes" schemename="dnet:access_modes"/>
+
+					#foreach($ctx in $util.processContexts($!pub.contexts))
+						#context($ctx)
+					#end
+
+					<datainfo>
+						<inferred>false</inferred>
+						<deletedbyinference>false</deletedbyinference>
+						<trust>0.9</trust>
+						<inferenceprovenance/>
+						<provenanceaction classid="user:insert" classname="user:insert" schemeid="dnet:provenanceActions" schemename="dnet:provenanceActions"/>
+					</datainfo>
+
+					<rels>
+						#foreach($author in $!pub.authors)
+							<rel inferred="false" trust="0.9" inferenceprovenance="" provenanceaction="sysimport:crosswalk:repository">
+								<to class="hasAuthor" scheme="dnet:personroles" type="person"/>
+								<ranking>$velocityCount</ranking>
+								<fullname>$!author</fullname>
+							</rel>
+						#end
+						#foreach($link in $!pub.linksToProjects)
+
+							#set( $info = $!util.calculateProjectInfo($!link) )
+							<rel inferred="false" trust="0.9" inferenceprovenance="" provenanceaction="user:claim:search">
+								<to class="isProducedBy" scheme="dnet:result_project_relations" type="project">$!esc.xml($!info.id)</to>
+								<code>$!esc.xml($!info.code)</code>
+								<acronym>$!esc.xml($!info.acronym)</acronym>
+								<title>$!esc.xml($!info.title)</title>
+								<contracttype classid="" classname="" schemeid="" schemename=""/>
+
+								<funding>
+									<funder id="$!esc.xml($!info.funderId)"
+									        shortname="$!esc.xml($!info.funderShortName)"
+									        name="$!esc.xml($!info.funderName)"
+									        jurisdiction="$!esc.xml($!info.jurisdiction)"/>
+									<funding_level_0 name="$!esc.xml($!info.fundingName)">$!esc.xml($!info.fundingId)</funding_level_0>
+								</funding>
+							</rel>
+						#end
+
+					</rels>
+					<children>
+						<instance id="$!esc.xml($!objIdentifier)">
+
+							<licence classid="$!esc.xml($!pub.licenseCode)" classname="$!esc.xml($licenses.get($!pub.licenseCode))"
+							         schemeid="dnet:access_modes" schemename="dnet:access_modes"/>
+
+							<instancetype classid="$!esc.xml($!pub.resourceType)" classname="$!esc.xml($resourceTypes.get($!pub.resourceType))"
+							              schemeid="dnet:publication_resource" schemename="dnet:publication_resource"/>
+							<hostedby name="$!esc.xml($!hostedBy.name)" id="$!esc.xml($!hostedBy.calculateOpenaireId())"/>
+							<webresource>
+								<url>$!esc.xml($!pub.url)</url>
+							</webresource>
+						</instance>
+					</children>
+				</oaf:result>
+			</oaf:entity>
+		</metadata>
+	</result>
+</record>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/vocabulary/ec4a6f99-bbc1-482d-aa3a-c171f9401d26.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/vocabulary/ec4a6f99-bbc1-482d-aa3a-c171f9401d26.xml	(revision 43443)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/vocabulary/ec4a6f99-bbc1-482d-aa3a-c171f9401d26.xml	(revision 43444)
@@ -39,6 +39,10 @@
 					<SYNONYMS/>
 					<RELATIONS/>
 				</TERM>
+				<TERM native_name="CMREntityRegistry" code="CMREntityRegistry" english_name="CMREntityRegistry" encoding="OPENAIRE">
+					<SYNONYMS/>
+					<RELATIONS/>
+				</TERM>
 				<TERM native_name="fileCSV" code="fileCSV" english_name="fileCSV" encoding="OPENAIRE">
 					<SYNONYMS/>
 					<RELATIONS/>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/TransformationRuleDSResources/TransformationRuleDSResourceType/CMR_EntityRegistry2db.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/TransformationRuleDSResources/TransformationRuleDSResourceType/CMR_EntityRegistry2db.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/TransformationRuleDSResources/TransformationRuleDSResourceType/CMR_EntityRegistry2db.xml	(revision 43444)
@@ -0,0 +1,48 @@
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="71216e7b-eb38-4e41-b34a-d9d8fa71950d_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU="/>
+		<RESOURCE_TYPE value="TransformationRuleDSResourceType"/>
+		<RESOURCE_KIND value="TransformationRuleDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2014-07-17T10:41:08+02:00"/>
+	</HEADER>
+	<BODY>
+		<CONFIGURATION>
+			<IMPORTED/>
+			<SCRIPT>
+				<TITLE>CMR EntityRegisty 2 DB</TITLE>
+				<CODE><![CDATA[<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+    xmlns:xs="http://www.w3.org/2001/XMLSchema"
+    exclude-result-prefixes="xs"
+    version="2.0">
+    <xsl:param name="parentDatasourceId"/>
+    <xsl:param name="namespacePrefix"/>
+
+    <xsl:template match="/">
+        <record>
+            <xsl:copy-of select=".//*[local-name()='header']"/>
+            <metadata>
+                <ROWS>
+                    <ROW table="datasources">
+                        <FIELD name="id"><xsl:value-of select="concat($namespacePrefix,'::', //datasource/id)"/></FIELD>
+                        <FIELD name="_dnet_resource_identifier_"><xsl:value-of select="concat($namespacePrefix,'::', //datasource/id)"/></FIELD>
+                        <FIELD name="officialname"><xsl:value-of select="//datasource/name"/></FIELD>
+                        <FIELD name="englishname"><xsl:value-of select="//datasource/name"/></FIELD>
+                        <FIELD name="od_contenttypes">dataset</FIELD>
+                        <FIELD name="websiteurl"><xsl:value-of select="//datasource/url"/></FIELD>
+                        <FIELD name="namespaceprefix"><xsl:value-of select="concat('cmr_',substring(//datasource/uuid,1,8))"/></FIELD>
+                        <FIELD name="datasourceclass">datarepository</FIELD>
+                        <FIELD name="collectedfrom"><xsl:value-of select="$parentDatasourceId"/></FIELD>
+                    </ROW>
+                </ROWS>
+            </metadata>
+        </record>
+    </xsl:template>
+</xsl:stylesheet>
+]]></CODE>
+			</SCRIPT>
+		</CONFIGURATION>
+		<STATUS/>
+		<SECURITY_PARAMETERS/>
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/closeMesh.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/closeMesh.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/closeMesh.xml	(revision 43444)
@@ -0,0 +1,153 @@
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="9522876c-5885-4dd2-9d06-92e9a3eaa9f1_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2006-05-04T18:13:51.0Z"/>
+	</HEADER>
+	<BODY>
+		<CONFIGURATION>
+
+			<PARAMETERS>
+				<PARAM name="hbaseTable" description="HBase table name" required="true" type="string"/>
+				<PARAM name="cluster" description="Hadoop cluster name" required="true" type="string"/>
+				<PARAM name="workDir" description="minDist algorithm work directory" required="true" type="string"/>
+				<PARAM name="entitySequence" description="list of entity names to be considered by the algorithm" required="true" type="string" default="organization,result"/>
+			</PARAMETERS>
+
+			<WORKFLOW>
+
+				<NODE name="fetchRelClasses" type="FetchRelClasses" isStart="true">
+					<DESCRIPTION/>
+					<PARAMETERS>
+						<PARAM name="xquery" property="dnet.openaire.model.relclasses.xquery"/>
+						<PARAM name="relClassesName" value="relClasses"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="entitySequence"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="entitySequence" type="CheckEntitySequenceJob">
+					<DESCRIPTION>Check entity sequence</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="entitySequence" ref="entitySequence"/>
+						<PARAM name="dedupConfigSequenceParam" value="dedup.conf.queue"/>
+						<PARAM name="entityTypeNameParam" value="dedup.entity.name"/>
+						<PARAM name="entityTypeIdParam" value="dedup.entity.id"/>
+						<PARAM name="actionSetParam" value="actionset"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="resetPath"/>
+						<ARC name="done" to="success"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="resetPath" type="CreateHdfsDirJob">
+					<DESCRIPTION>input files cleanup</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="force" value="true"/>
+						<PARAM name="path" ref="workDir"/>
+						<PARAM name="cluster" ref="cluster"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="mindist"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="mindist" type="MinDistSearchHadoopJob">
+					<DESCRIPTION>find the minimum vertex in each adjacency lists</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="debug" value="false"/>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="outPathParam" value="outputPath"/>
+						<PARAM name="dedupConfigSequence" env="actionset"/>
+						<PARAM name="workDir" ref="workDir"/>
+						<PARAM name="jobParams">
+							<MAP>
+								<ENTRY key="cluster" ref="cluster"/>
+								<ENTRY key="entityTypeId" env="dedup.entity.id"/>
+								<ENTRY key="hbase.mapred.inputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapreduce.inputtable" ref="hbaseTable"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC name="depth_n" to="mindist"/>
+						<ARC to="components"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="components" type="DedupConfigurationLoaderJob">
+					<DESCRIPTION>joins all the vertex ids to build the connected components in the graph</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="hadoopJob" value="connectedComponentsJob"/>
+						<PARAM name="dedupConfigSequence" env="actionset"/>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="jobParams">
+							<MAP>
+								<ENTRY key="hbase.mapred.outputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapreduce.outputtable" ref="hbaseTable"/>
+								<ENTRY key="entityType" env="dedup.entity.name"/>
+								<ENTRY key="entityTypeId" env="dedup.entity.id"/>
+								<ENTRY key="mapred.input.dir" env="outputPath"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="markDeleted"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="markDeleted" type="DedupConfigurationLoaderJob">
+					<DESCRIPTION>mark duplicates as deleted by inference</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="hadoopJob" value="dedupMarkDeletedEntityJob"/>
+						<PARAM name="dedupConfigSequence" env="actionset"/>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="jobParams">
+							<MAP>
+								<ENTRY key="hbase.mapred.inputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapreduce.inputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapred.outputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapreduce.outputtable" ref="hbaseTable"/>
+								<ENTRY key="entityType" env="dedup.entity.name"/>
+								<ENTRY key="entityTypeId" env="dedup.entity.id"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="buildRoots"/>
+					</ARCS>
+				</NODE>
+
+
+				<NODE name="buildRoots" type="DedupConfigurationLoaderJob">
+					<DESCRIPTION>redirect rels</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="hadoopJob" value="dedupBuildRootsJob"/>
+						<PARAM name="dedupConfigSequence" env="actionset"/>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="jobParams">
+							<MAP>
+								<ENTRY key="hbase.mapred.inputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapred.outputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapreduce.inputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapreduce.outputtable" ref="hbaseTable"/>
+								<ENTRY key="entityType" env="dedup.entity.name"/>
+								<ENTRY key="entityTypeId" env="dedup.entity.id"/>
+								<ENTRY key="relClasses" env="relClasses"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="entitySequence"/>
+					</ARCS>
+				</NODE>
+
+
+			</WORKFLOW>
+		</CONFIGURATION>
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/updateProfiles.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/updateProfiles.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/updateProfiles.xml	(revision 43444)
@@ -0,0 +1,68 @@
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="629ccae5-6a34-4df3-9569-d0ae037e2bc0_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl"/>
+		<RESOURCE_TYPE value="WorkflowDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2016-07-19T18:13:51.0Z"/>
+	</HEADER>
+	<BODY>
+		<WORKFLOW_NAME menuSection="InfoSpace Consistency">Update datasource profiles</WORKFLOW_NAME>
+		<WORKFLOW_DESCRIPTION>Update datasource profiles from sql database</WORKFLOW_DESCRIPTION>
+		<WORKFLOW_INFO/>
+		<WORKFLOW_FAMILY>UpdateDB</WORKFLOW_FAMILY>
+		<WORKFLOW_PRIORITY>30</WORKFLOW_PRIORITY>
+
+		<CONFIGURATION status="EXECUTABLE" start="MANUAL">
+			<PARAMETERS/>
+			<WORKFLOW>
+
+
+				<NODE isStart="true" name="findComplianceRepos" type="FindComplianceRepositories">
+					<DESCRIPTION/>
+					<PARAMETERS>
+						<PARAM name="dbName" value="dnet_wds"/>
+						<PARAM name="eprParam" value="repoEpr"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="waitRepos"/>
+					</ARCS>
+				</NODE>
+				<NODE isStart="true" name="findExistingRepos" type="FindExistingRepositories">
+					<DESCRIPTION/>
+					<PARAMETERS>
+						<PARAM name="existingReposParam" value="existingRepos"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="waitRepos"/>
+					</ARCS>
+				</NODE>
+				<NODE isJoin="true" name="waitRepos">
+					<DESCRIPTION/>
+					<PARAMETERS/>
+					<ARCS>
+						<ARC to="updateRepoProfiles"/>
+					</ARCS>
+				</NODE>
+				<NODE name="updateRepoProfiles" type="UpdateRepositoryProfiles">
+					<DESCRIPTION/>
+					<PARAMETERS>
+						<PARAM name="eprParam" value="repoEpr"/>
+						<PARAM name="existingReposParam" value="existingRepos"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="success"/>
+					</ARCS>
+				</NODE>
+			</WORKFLOW>
+
+		</CONFIGURATION>
+		<NOTIFICATIONS/>
+		<SCHEDULING enabled="false">
+			<CRON>29 5 22 ? * *</CRON>
+			<MININTERVAL>10080</MININTERVAL>
+		</SCHEDULING>
+		<STATUS/>
+
+	</BODY>
+</RESOURCE_PROFILE>
\ No newline at end of file
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/update.global.OAI.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/update.global.OAI.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/update.global.OAI.xml	(revision 43444)
@@ -0,0 +1,67 @@
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="303a9a13-590d-4e70-8892-8173a6c6dffb_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl"/>
+		<RESOURCE_TYPE value="WorkflowDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2006-05-04T18:13:51.0Z"/>
+	</HEADER>
+	<BODY>
+		<WORKFLOW_NAME menuSection="InfoSpace Provision">Update OAI store</WORKFLOW_NAME>
+		<WORKFLOW_DESCRIPTION>Update OAI store</WORKFLOW_DESCRIPTION>
+		<WORKFLOW_INFO/>
+		<WORKFLOW_FAMILY>Data Provision</WORKFLOW_FAMILY>
+		<WORKFLOW_PRIORITY>35</WORKFLOW_PRIORITY>
+		<!-- <ADMIN_EMAIL>alessia.bardi@isti.cnr.it,claudio.atzori@isti.cnr.it</ADMIN_EMAIL> -->
+		<CONFIGURATION status="EXECUTABLE" start="MANUAL">
+
+			<PARAMETERS>
+				<PARAM name="cluster" description="Hadoop cluster name" required="true" type="string" managedBy="system">DM</PARAM>
+				<PARAM name="inputRecordsPath" description="path on HDFS pointing to the sequence file with the input xml records" required="true" type="string" managedBy="user"/>
+
+				<PARAM name="oaiDBName" description="db name" required="true" type="string" managedBy="user">oaistore</PARAM>
+				<PARAM name="oaiCollectionName" description="collection name" required="true" type="string" managedBy="system">oaf-index-openaire</PARAM>
+				<PARAM name="skipDuplicates" description="skip duplicates" required="true" type="boolean" managedBy="user">true</PARAM>
+				<PARAM name="duplicateXPath" description="xpath to identify duplicated records" required="true" type="string" managedBy="user">//*[local-name()='entity']//*[local-name()='datainfo']/*[local-name()='deletedbyinference'][./text() = 'true']</PARAM>
+
+				<PARAM name="fieldNames" description="list of field names to index" required="true" type="string" managedBy="user">set,datestamp;license,oaftype;set,oaftype;oaftype,funder;resulttypeid,oaftype;oaftype,funding;resulttypeid,funder;resulttypeid,funding</PARAM>
+				<PARAM name="configuredOnly" description="rebuild only the indicies in the OAI configuration profile" required="true" type="boolean" managedBy="user">false</PARAM>
+			</PARAMETERS>
+			<WORKFLOW>
+
+				<NODE name="updateOAI" type="LaunchWorkflowTemplate" isStart="true">
+					<DESCRIPTION>update OAI store</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="12f1db9b-3c3e-4fd6-a425-932afd6ef3be_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="cluster" ref="cluster"/>
+								<ENTRY key="inputRecordsPath" ref="inputRecordsPath"/>
+								<ENTRY key="oaiDBName" ref="oaiDBName"/>
+								<ENTRY key="oaiCollectionName" ref="oaiCollectionName"/>
+								<ENTRY key="skipDuplicates" ref="skipDuplicates"/>
+								<ENTRY key="duplicateXPath" ref="duplicateXPath"/>
+								<ENTRY key="fieldNames" ref="fieldNames"/>
+								<ENTRY key="configuredOnly" ref="configuredOnly"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="success"/>
+					</ARCS>
+				</NODE>
+
+			</WORKFLOW>
+
+		</CONFIGURATION>
+
+		<NOTIFICATIONS/>
+
+		<SCHEDULING enabled="false">
+			<CRON>29 5 22 ? * *</CRON>
+			<MININTERVAL>10080</MININTERVAL>
+		</SCHEDULING>
+
+		<STATUS/>
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/dedupPerson.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/dedupPerson.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/dedupPerson.xml	(revision 43444)
@@ -0,0 +1,102 @@
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="5e3e809c-c967-4c68-95f4-9cb8a503bbc3_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2006-05-04T18:13:51.0Z"/>
+	</HEADER>
+	<BODY>
+		<CONFIGURATION>
+
+			<PARAMETERS>
+				<PARAM name="hbaseTable" description="HBase table name" required="true" type="string"/>
+				<PARAM name="cluster" description="Hadoop cluster name" required="true" type="string"/>
+
+				<PARAM name="dedupConfig" description="deduplication configuration name " required="true" type="string"/>
+			</PARAMETERS>
+
+			<WORKFLOW>
+
+				<NODE isStart="true" name="fetchRelClasses" type="FetchRelClasses">
+					<DESCRIPTION/>
+					<PARAMETERS>
+						<PARAM name="xquery" property="dnet.openaire.model.relclasses.xquery"/>
+						<PARAM name="relClassesName" value="relClasses"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="waitConf"/>
+					</ARCS>
+				</NODE>
+
+
+				<NODE isStart="true" name="setDedupConfigs" type="SetDedupConfiguration">
+					<DESCRIPTION>Set Dedup conf</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="dedupConfigSequence" ref="dedupConfig"/>
+						<PARAM name="dedupConfigSequenceParam" value="dedup.conf.queue"/>
+						<PARAM name="entityTypeNameParam" value="dedup.entity.name"/>
+						<PARAM name="entityTypeIdParam" value="dedup.entity.id"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="waitConf"/>
+					</ARCS>
+				</NODE>
+
+				<NODE isJoin="true" name="waitConf">
+					<DESCRIPTION/>
+					<PARAMETERS/>
+					<ARCS>
+						<ARC to="dedupPhase"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="dedupPhase" type="DedupConfigurationAwareJob">
+					<DESCRIPTION>Dedup M/R job</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="hadoopJob" value="dedupPersonJob"/>
+						<PARAM name="dedupConfigurationOrchestration" env="dedup.conf.queue"/>
+						<PARAM name="jobParams">
+							<MAP>
+								<ENTRY key="hbase.mapred.inputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapred.outputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapreduce.inputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapreduce.outputtable" ref="hbaseTable"/>
+								<ENTRY key="entityType" env="dedup.entity.name"/>
+								<ENTRY key="entityTypeId" env="dedup.entity.id"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="buildRoots"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="buildRoots" type="DedupConfigurationLoaderJob">
+					<DESCRIPTION>redirect rels</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="hadoopJob" value="dedupBuildRootsJob"/>
+						<PARAM name="dedupConfigSequence" ref="dedupConfig"/>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="jobParams">
+							<MAP>
+								<ENTRY key="hbase.mapred.inputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapred.outputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapreduce.inputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapreduce.outputtable" ref="hbaseTable"/>
+								<ENTRY key="entityType" env="dedup.entity.name"/>
+								<ENTRY key="entityTypeId" env="dedup.entity.id"/>
+								<ENTRY key="relClasses" env="relClasses"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="success"/>
+					</ARCS>
+				</NODE>
+
+			</WORKFLOW>
+		</CONFIGURATION>
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/sql2hbase.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/sql2hbase.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/sql2hbase.xml	(revision 43444)
@@ -0,0 +1,66 @@
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="9bb04513-be1d-4d13-8e86-785cc0375635_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2006-05-04T18:13:51.0Z"/>
+	</HEADER>
+	<BODY>
+		<CONFIGURATION>
+
+			<PARAMETERS>
+				<PARAM name="db" description="relational db name" required="true" type="string"/>
+				<PARAM name="sqlQuery" description="the SQL query" required="true" type="string"/>
+				<PARAM name="sqlForSize" description="the SQL query used to determine the row set size" required="false" type="string"/>
+				<PARAM name="sqlPrepareQuery" description="the prepare SQL query" required="false" type="string"/>
+
+				<PARAM name="hbaseTable" description="HBase table name" required="true" type="string"/>
+				<PARAM name="cluster" description="Hadoop cluster name" required="true" type="string"/>
+
+				<PARAM name="mapping" description="xslt mapping function" required="true" type="string"/>
+			</PARAMETERS>
+
+			<WORKFLOW>
+
+				<NODE name="prepareQuery" type="ExecuteSql" isStart="true">
+					<DESCRIPTION>prepare query</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="db" ref="db"/>
+						<PARAM name="sql" ref="sqlPrepareQuery"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="queryDB"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="queryDB" type="QueryDb">
+					<DESCRIPTION>perform SQL query</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="db" ref="db"/>
+						<PARAM name="sql" ref="sqlQuery"/>
+						<PARAM name="sqlForSize" ref="sqlForSize"/>
+						<PARAM name="outputEprParam" value="outputResultSet"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="storeResultSet"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="storeResultSet" type="StoreHBase">
+					<DESCRIPTION>Store ResultSet to HBase</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="inputEprParam" value="outputResultSet"/>
+						<PARAM name="hbaseTable" ref="hbaseTable"/>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="mapping" ref="mapping"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="success"/>
+					</ARCS>
+				</NODE>
+
+			</WORKFLOW>
+		</CONFIGURATION>
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/provision.global.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/provision.global.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/provision.global.xml	(revision 43444)
@@ -0,0 +1,267 @@
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="b6addffa-bae1-42db-811c-ae20a5c514a6_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl"/>
+		<RESOURCE_TYPE value="WorkflowDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2006-05-04T18:13:51.0Z"/>
+	</HEADER>
+	<BODY>
+		<WORKFLOW_NAME menuSection="InfoSpace Provision">OpenAIRE Provision</WORKFLOW_NAME>
+		<WORKFLOW_DESCRIPTION>OpenAIRE Data Provision</WORKFLOW_DESCRIPTION>
+		<WORKFLOW_INFO/>
+		<WORKFLOW_FAMILY>Data Provision</WORKFLOW_FAMILY>
+		<WORKFLOW_PRIORITY>35</WORKFLOW_PRIORITY>
+		<!-- <ADMIN_EMAIL>alessia.bardi@isti.cnr.it,claudio.atzori@isti.cnr.it</ADMIN_EMAIL> -->
+		<CONFIGURATION status="EXECUTABLE" start="MANUAL">
+
+			<PARAMETERS>
+				<PARAM name="table" description="HBase table to be dropped" required="true" managedBy="user" type="string"></PARAM>
+				<PARAM name="cluster" description="Hadoop cluster logical name" required="true" managedBy="user" type="string" function="validValues(['DM','IIS'])"></PARAM>
+				<PARAM name="reuseRegionInfo" description="Reuse table splits" required="true" managedBy="user" type="boolean">true</PARAM>
+
+				<PARAM name="tableConf" description="HBase table configuration param name" required="true" managedBy="system" type="string">hbase.table.conf</PARAM>
+				<PARAM name="tableCols" description="HBase table schema param name" required="true" managedBy="system" type="string">hbase.table.cols</PARAM>
+
+				<PARAM name="db" description="relational db name" required="true" type="string" managedBy="user"/>
+
+				<PARAM name="mappingDS" description="xslt mapping function" required="true" type="string" managedBy="user" function="obtainValues('dbmf2hbaseMappings', {})"/>
+				<PARAM name="mappingDS_ORG" description="xslt mapping function" required="true" type="string" managedBy="user" function="obtainValues('dbmf2hbaseMappings', {})"/>
+				<PARAM name="mappingORG" description="xslt mapping function" required="true" type="string" managedBy="user" function="obtainValues('dbmf2hbaseMappings', {})"/>
+				<PARAM name="mappingPERSON" description="xslt mapping function" required="true" type="string" managedBy="user" function="obtainValues('dbmf2hbaseMappings', {})"/>
+				<PARAM name="mappingPRJ_ORG" description="xslt mapping function" required="true" type="string" managedBy="user" function="obtainValues('dbmf2hbaseMappings', {})"/>
+				<PARAM name="mappingPRJ_PERSON" description="xslt mapping function" required="true" type="string" managedBy="user" function="obtainValues('dbmf2hbaseMappings', {})"/>
+				<PARAM name="mappingPRJ" description="xslt mapping function" required="true" type="string" managedBy="user" function="obtainValues('dbmf2hbaseMappings', {})"/>
+				<PARAM name="mappingCLAIM_RELS" description="xslt mapping function" required="true" type="string" managedBy="user" function="obtainValues('dbmf2hbaseMappings', {})"/>
+
+
+				<PARAM name="mappingPublication" description="xslt mapping for publications" required="true" type="string" managedBy="user" function="obtainValues('oaf2hbaseMappings', {})"/>
+				<PARAM name="reusePublication" description="reuse publications on HDFS?" required="true" type="boolean" managedBy="user"/>
+				<PARAM name="hdfsPathPublication" description="hdfs path for publications" required="true" type="string" managedBy="user"/>
+
+				<PARAM name="mappingDataset" description="xslt mapping for datasets" required="true" type="string" managedBy="user" function="obtainValues('odf2hbaseMappings', {})"/>
+				<PARAM name="reuseDataset" description="reuse datasets on HDFS?" required="true" type="boolean" managedBy="user"/>
+				<PARAM name="hdfsPathDataset" description="hdfs path for datasets" required="true" type="string" managedBy="user"/>
+
+				<PARAM name="mappingSimilarities" description="xslt mapping function for similarities" required="true" type="string" managedBy="user" function="obtainValues('dbmf2hbaseMappings', {})"/>
+				<PARAM name="mappingDissimilarities" description="xslt mapping function for dissimilarities" required="true" type="string" managedBy="user" function="obtainValues('dbmf2hbaseMappings', {})"/>
+				<PARAM name="actionSets" description="csv list of action sets to be promoted" required="true" type="string" managedBy="user"/>
+
+				<PARAM name="personDedupConfig" description="dedup person configuration" required="true" type="string" managedBy="user" function="obtainValues('dedupOrchestrations', {})"/>
+				<PARAM name="minDistWorkDir" description="work directory for the minDist algorithm" required="true" type="string" managedBy="user"/>
+
+				<PARAM name="mdFormat" description="metadata format name" required="true" type="string" managedBy="user" function="validValues(['DMF','TMF'])"></PARAM>
+
+				<PARAM name="oaiDBName" description="OAI store db name" required="true" type="string" managedBy="user"/>
+
+			</PARAMETERS>
+			<WORKFLOW>
+
+				<NODE name="resetHbase" type="LaunchWorkflowTemplate" isStart="true">
+					<DESCRIPTION>Reset HBase table</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="75345aba-c069-43f4-90aa-e13688d9845e_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="table" ref="table"/>
+								<ENTRY key="cluster" ref="cluster"/>
+								<ENTRY key="reuseRegionInfo" ref="reuseRegionInfo"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+
+					<ARCS>
+						<ARC to="db2hbase"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="db2hbase" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>load db to hbase</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="3ffe4cd8-73f1-4566-85fc-26ac32e8af56_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="db" ref="db"/>
+								<ENTRY key="hbaseTable" ref="table"/>
+								<ENTRY key="cluster" ref="cluster"/>
+
+								<ENTRY key="sqlQueryDS" value="/eu/dnetlib/msro/workflows/hbase/queryDatasources.sql"/>
+								<ENTRY key="mappingDS" ref="mappingDS"/>
+
+								<ENTRY key="sqlQueryDS_ORG" value="/eu/dnetlib/msro/workflows/hbase/queryDatasourceOrganization.sql"/>
+								<ENTRY key="mappingDS_ORG" ref="mappingDS_ORG"/>
+
+								<ENTRY key="sqlQueryORG" value="/eu/dnetlib/msro/workflows/hbase/queryOrganizations.sql"/>
+								<ENTRY key="mappingORG" ref="mappingORG"/>
+
+								<ENTRY key="sqlQueryPERSON" value="/eu/dnetlib/msro/workflows/hbase/queryPersons.sql"/>
+								<ENTRY key="mappingPERSON" ref="mappingPERSON"/>
+
+								<ENTRY key="sqlQueryPRJ_ORG" value="/eu/dnetlib/msro/workflows/hbase/queryProjectOrganization.sql"/>
+								<ENTRY key="mappingPRJ_ORG" ref="mappingPRJ_ORG"/>
+
+								<ENTRY key="sqlQueryPRJ_PERSON" value="/eu/dnetlib/msro/workflows/hbase/queryProjectContactperson.sql"/>
+								<ENTRY key="mappingPRJ_PERSON" ref="mappingPRJ_PERSON"/>
+
+								<ENTRY key="sqlQueryPRJ" value="/eu/dnetlib/msro/workflows/hbase/queryProjects.sql"/>
+								<ENTRY key="mappingPRJ" ref="mappingPRJ"/>
+
+								<ENTRY key="sqlQueryCLAIM_RELS" value="/eu/dnetlib/msro/workflows/hbase/queryClaimsRel.sql"/>
+								<ENTRY key="mappingCLAIM_RELS" ref="mappingCLAIM_RELS"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+
+					<ARCS>
+						<ARC to="oaf2hbase"/>
+					</ARCS>
+
+				</NODE>
+
+				<NODE name="oaf2hbase" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>OAF Publications to HBase</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="5b05a65a-4eeb-4862-bc55-b35c7ec3baf0_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="hbaseTable" ref="table"/>
+								<ENTRY key="cluster" ref="cluster"/>
+								<ENTRY key="mdFormat" value="OAF"/>
+								<ENTRY key="mapping" ref="mappingPublication"/>
+								<ENTRY key="reuseMdRecords" ref="reusePublication"/>
+								<ENTRY key="hdfsPath" ref="hdfsPathPublication"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="odf2hbase"/>
+					</ARCS>
+				</NODE>
+
+
+				<NODE name="odf2hbase" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>ODF Datasets to HBase</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="5b05a65a-4eeb-4862-bc55-b35c7ec3baf0_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="hbaseTable" ref="table"/>
+								<ENTRY key="cluster" ref="cluster"/>
+								<ENTRY key="mdFormat" value="ODF"/>
+								<ENTRY key="mapping" ref="mappingDataset"/>
+								<ENTRY key="reuseMdRecords" ref="reuseDataset"/>
+								<ENTRY key="hdfsPath" ref="hdfsPathDataset"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="actions2hbase"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="actions2hbase" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>actions to hbase</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="72a974b4-a13f-48e2-a18c-98e0effb4579_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="hbaseTable" ref="table"/>
+								<ENTRY key="cluster" ref="cluster"/>
+								<ENTRY key="mappingSimilarities" ref="mappingSimilarities"/>
+								<ENTRY key="mappingDissimilarities" ref="mappingDissimilarities"/>
+								<ENTRY key="set" ref="actionSets"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="dedupPerson"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="dedupPerson" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>dedup person</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="5e3e809c-c967-4c68-95f4-9cb8a503bbc3_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="hbaseTable" ref="table"/>
+								<ENTRY key="cluster" ref="cluster"/>
+								<ENTRY key="dedupConfig" ref="personDedupConfig"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+
+					<ARCS>
+						<ARC to="closeMesh"/>
+					</ARCS>
+				</NODE>
+
+
+				<NODE name="closeMesh" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>close mesh</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="9522876c-5885-4dd2-9d06-92e9a3eaa9f1_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="hbaseTable" ref="table"/>
+								<ENTRY key="cluster" ref="cluster"/>
+								<ENTRY key="workDir" ref="minDistWorkDir"/>
+								<ENTRY key="entitySequence" value="organization,result"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+
+					<ARCS>
+						<ARC to="applyBlacklist"/>
+					</ARCS>
+				</NODE>
+
+
+				<NODE name="applyBlacklist">
+					<DESCRIPTION>Resolve the identifiers in the blacklist</DESCRIPTION>
+					<PARAMETERS/>
+					<ARCS>
+						<ARC to="updateProvision"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="updateProvision" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>update provision</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="a4434d62-d4cd-4c73-a107-bc7c62e6f815_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="mdFormat" ref="mdFormat"/>
+								<ENTRY key="hbaseTable" ref="table"/>
+								<ENTRY key="cluster" ref="cluster"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="preparePrepublic"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="preparePrepublic">
+					<DESCRIPTION>prepare pre-public infospace</DESCRIPTION>
+					<PARAMETERS/>
+
+					<ARCS>
+						<ARC to="success"/>
+					</ARCS>
+				</NODE>
+
+			</WORKFLOW>
+
+		</CONFIGURATION>
+
+		<NOTIFICATIONS/>
+
+		<SCHEDULING enabled="false">
+			<CRON>29 5 22 ? * *</CRON>
+			<MININTERVAL>10080</MININTERVAL>
+		</SCHEDULING>
+
+		<STATUS/>
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/dedup.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/dedup.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/dedup.xml	(revision 43444)
@@ -0,0 +1,232 @@
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="01ed11e8-e874-4478-a8ac-83e63e9699e4_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2016-06-25T10:24:32+00:00"/>
+	</HEADER>
+	<BODY>
+		<CONFIGURATION>
+
+			<PARAMETERS>
+
+				<PARAM name="hbaseTable" description="HBase table name" required="true" type="string"/>
+				<PARAM name="cluster" description="Hadoop cluster name" required="true" type="string"/>
+
+				<PARAM name="dedupConfigSequence" description="dedup configuration orchestration name" required="true" type="string"/>
+				<PARAM name="minDistWorkDir" description="work directory for the minDist algorithm" required="true" type="string"/>
+
+				<PARAM name="mappingSimilarities" description="xslt mapping function for similarities" required="true" type="string"/>
+				<PARAM name="mappingDissimilarities" description="xslt mapping function for dissimilarities" required="true" type="string"/>
+
+				<PARAM name="mdFormat" description="dedup index mdFormat" required="true" type="string" default="OPENAIRE"/>
+				<PARAM name="layout" description="dedup index layout" required="true" type="string" default="index"/>
+				<PARAM name="interpretation" description="dedup index interpretation" required="true" type="string" default="dedup"/>
+
+			</PARAMETERS>
+
+			<WORKFLOW>
+
+				<NODE isStart="true" name="fetchRelClasses" type="FetchRelClasses">
+					<DESCRIPTION/>
+					<PARAMETERS>
+						<PARAM name="xquery" property="dnet.openaire.model.relclasses.xquery"/>
+						<PARAM name="relClassesName" value="relClasses"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="resetPath"/>
+					</ARCS>
+				</NODE>
+
+				<NODE isStart="true" name="setDedupConfigs" type="SetDedupConfiguration">
+					<DESCRIPTION>Set Dedup conf</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="dedupConfigSequence" ref="dedupConfigSequence"/>
+						<PARAM name="dedupConfigSequenceParam" value="dedup.conf.queue"/>
+						<PARAM name="entityTypeNameParam" value="entityType"/>
+						<PARAM name="entityTypeIdParam" value="entityTypeId"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="resetPath"/>
+					</ARCS>
+				</NODE>
+
+
+				<NODE name="resetPath" type="CreateHdfsDirJob" isJoin="true">
+					<DESCRIPTION>input files cleanup</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="force" value="true"/>
+						<PARAM name="path" ref="minDistWorkDir"/>
+						<PARAM name="cluster" ref="cluster"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="deduplicateScan"/>
+					</ARCS>
+				</NODE>
+
+
+				<NODE name="deduplicateScan" type="DuplicateScanJob">
+					<DESCRIPTION>Dup Scan</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="hadoopJob" value="dedupCandidateScanJob"/>
+						<PARAM name="dedupConfigurationOrchestration" env="dedup.conf.queue"/>
+						<PARAM name="jobParams">
+							<MAP>
+								<ENTRY key="entityTypeId" env="entityTypeId"/>
+								<ENTRY key="entityType" env="entityType"/>
+								<ENTRY key="cluster" ref="cluster"/>
+								<ENTRY key="hbase.mapred.inputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapred.outputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapreduce.inputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapreduce.outputtable" ref="hbaseTable"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="deduplicateScan"/>
+						<ARC name="done" to="prepareActionSet"/>
+						<ARC name="done" to="queryUserSimilarities"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="prepareActionSet" type="PrepareConfiguredActionSet">
+					<DESCRIPTION>prepare action sets</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="dedupConfigSequence" ref="dedupConfigSequence"/>
+						<PARAM name="jobProperty" value="rawSetId"/>
+						<PARAM name="actionSetPathParam" value="actionSetPath"/>
+						<PARAM name="setsParam" value="sets"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="similarity2actions"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="similarity2actions" type="DedupSimilarityToActions">
+					<DESCRIPTION>export the similarity rels as Actions</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="hadoopJob" value="dedupSimilarity2HdfsActionsJob"/>
+						<PARAM name="dedupConfigSequence" ref="dedupConfigSequence"/>
+						<PARAM name="entityType" env="entityType"/>
+						<PARAM name="jobParams">
+							<MAP>
+								<ENTRY key="dedup.conf" env="dedup.conf"/>
+								<ENTRY key="entityTypeId" env="entityTypeId"/>
+								<ENTRY key="entityType" env="entityType"/>
+								<ENTRY key="cluster" ref="cluster"/>
+								<ENTRY key="rawSetId" env="rawSetId"/>
+								<ENTRY key="mapred.output.dir" env="actionSetPath"/>
+								<ENTRY key="hbase.mapred.inputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapreduce.inputtable" ref="hbaseTable"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="updateActionSets"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="updateActionSets" type="UpdateActionSets">
+					<DESCRIPTION>update action sets</DESCRIPTION>
+					<PARAMETERS/>
+					<ARCS>
+						<ARC to="doneActions"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="queryUserSimilarities" type="QueryUserActionDbJob">
+					<DESCRIPTION>query user similarity</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="dedupConfigSequence" ref="dedupConfigSequence"/>
+						<PARAM name="db" property="dnet.dedup.db.name"/>
+						<PARAM name="sql" value="/eu/dnetlib/msro/workflows/dedup/querySimilaritiesBySet.sql.st"/>
+						<PARAM name="outputEprParam" value="simEpr"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="buildSimilarityMesh"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="buildSimilarityMesh" type="BuildSimilarityMeshJob">
+					<DESCRIPTION>build mesh</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="inputEprParam" value="simEpr"/>
+						<PARAM name="outputEprParam" value="simMeshEpr"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="storeSimilarities"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="storeSimilarities" type="StoreHBase">
+					<DESCRIPTION>store similarity</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="inputEprParam" value="simMeshEpr"/>
+						<PARAM name="hbaseTable" ref="hbaseTable"/>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="mapping" ref="mappingSimilarities"/>
+						<PARAM name="simulation" value="false"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="queryUserDissimilarities"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="queryUserDissimilarities" type="QueryUserActionDbJob">
+					<DESCRIPTION>query user dissimilarity</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="dedupConfigSequence" ref="dedupConfigSequence"/>
+						<PARAM name="db" property="dnet.dedup.db.name"/>
+						<PARAM name="sql" value="/eu/dnetlib/msro/workflows/dedup/queryDissimilaritiesBySet.sql.st"/>
+						<PARAM name="outputEprParam" value="dissimEpr"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="storeDissimilarities"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="storeDissimilarities" type="DeleteFromHBase">
+					<DESCRIPTION>store dissimilarity</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="inputEprParam" value="dissimEpr"/>
+						<PARAM name="hbaseTable" ref="hbaseTable"/>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="mapping" ref="mappingDissimilarities"/>
+						<PARAM name="simulation" value="false"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="doneActions"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="doneActions" isJoin="true">
+					<DESCRIPTION>done actions</DESCRIPTION>
+					<PARAMETERS/>
+					<ARCS>
+						<ARC to="closeMesh"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="closeMesh" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>close mesh</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="9522876c-5885-4dd2-9d06-92e9a3eaa9f1_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="hbaseTable" ref="hbaseTable"/>
+								<ENTRY key="cluster" ref="cluster"/>
+								<ENTRY key="workDir" ref="minDistWorkDir"/>
+								<ENTRY key="entitySequence" env="entityType"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="success"/>
+					</ARCS>
+				</NODE>
+
+			</WORKFLOW>
+		</CONFIGURATION>
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/promoteActions.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/promoteActions.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/promoteActions.xml	(revision 43444)
@@ -0,0 +1,110 @@
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="72a974b4-a13f-48e2-a18c-98e0effb4579_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2006-05-04T18:13:51.0Z"/>
+	</HEADER>
+	<BODY>
+		<CONFIGURATION>
+
+			<PARAMETERS>
+
+				<PARAM name="set" description="csv list of action sets to be promoted" required="true" type="string"/>
+				<PARAM name="mappingSimilarities" description="xslt mapping function for similarities" required="true" type="string"/>
+				<PARAM name="mappingDissimilarities" description="xslt mapping function for dissimilarities" required="true" type="string"/>
+
+				<PARAM name="hbaseTable" description="HBase table name" required="true" type="string"/>
+				<PARAM name="cluster" description="Hadoop cluster name" required="true" type="string"/>
+			</PARAMETERS>
+
+			<WORKFLOW>
+
+				<NODE isStart="true" name="promoteActions" type="PromoteActionsHDFS">
+					<DESCRIPTION>Promote Actions</DESCRIPTION>
+					<PARAMETERS>
+						<!-- dedup-similarity-organization-simple,dedup-similarity-result-levenstein,iis-dataset-entities-main,iis-referenced-datasets-preprocessing,iis-dataset-entities-preprocessing,iis-document-citations,iis-document-classes,iis-referenced-projects-main,iis-document-similarities,iis-document-statistics,iis-extracted-metadata,iis-referenced-datasets-main,iis-referenced-projects-preprocessing,iis-researchinitiative,iis-wos-entities,iis-referenceextraction-pdb -->
+						<PARAM name="set" ref="set"/>
+						<PARAM name="tableName" ref="hbaseTable"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="querySimilarities"/>
+						<ARC to="queryDissimilarities"/>
+					</ARCS>
+				</NODE>
+
+
+				<NODE name="querySimilarities" type="QueryDb">
+					<DESCRIPTION>query similarity</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="db" property="dnet.dedup.db.name"/>
+						<PARAM name="sql" value="/eu/dnetlib/msro/workflows/dedup/querySimilarities.sql"/>
+						<PARAM name="outputEprParam" value="simEpr"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="buildSimilarityMesh"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="buildSimilarityMesh" type="BuildSimilarityMeshJob">
+					<DESCRIPTION>build mesh</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="inputEprParam" value="simEpr"/>
+						<PARAM name="outputEprParam" value="simMeshEpr"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="storeSimilarities"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="storeSimilarities" type="StoreHBase">
+					<DESCRIPTION>Store ResultSet to HBase</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="inputEprParam" value="simMeshEpr"/>
+						<PARAM name="hbaseTable" ref="hbaseTable"/>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="mapping" ref="mappingSimilarities"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="wait"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="queryDissimilarities" type="QueryDb">
+					<DESCRIPTION>query dissimilarity</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="db" property="dnet.dedup.db.name"/>
+						<PARAM name="sql" value="/eu/dnetlib/msro/workflows/dedup/queryDissimilarities.sql"/>
+						<PARAM name="outputEprParam" value="dissimEpr"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="storeDissimilarities"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="storeDissimilarities" type="DeleteFromHBase">
+					<DESCRIPTION>Store ResultSet to HBase</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="inputEprParam" value="dissimEpr"/>
+						<PARAM name="hbaseTable" ref="hbaseTable"/>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="mapping" ref="mappingDissimilarities"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="wait"/>
+					</ARCS>
+				</NODE>
+
+				<NODE isJoin="true" name="wait">
+					<DESCRIPTION/>
+					<PARAMETERS/>
+					<ARCS>
+						<ARC to="success"/>
+					</ARCS>
+				</NODE>
+
+			</WORKFLOW>
+		</CONFIGURATION>
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/dedup.update.index.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/dedup.update.index.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/dedup.update.index.xml	(revision 43444)
@@ -0,0 +1,140 @@
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="a1b0b30c-7e19-46e2-888c-67c0b880b346_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2016-06-25T10:24:32+00:00"/>
+	</HEADER>
+	<BODY>
+		<CONFIGURATION>
+
+			<PARAMETERS>
+				<PARAM name="hbaseTable" description="HBase table name" required="true" type="string"/>
+				<PARAM name="cluster" description="Hadoop cluster name" required="true" type="string"/>
+
+				<PARAM name="dedupConfigSequence" description="dedup configuration orchestration name" required="true" type="string"/>
+
+				<PARAM name="mdFormat" description="dedup index mdFormat" required="true" type="string" default="OPENAIRE"/>
+				<PARAM name="layout" description="dedup index layout" required="true" type="string" default="index"/>
+				<PARAM name="interpretation" description="dedup index interpretation" required="true" type="string" default="dedup"/>
+			</PARAMETERS>
+
+			<WORKFLOW>
+
+				<NODE isStart="true" name="setDedupConfigs" type="SetDedupConfiguration">
+					<DESCRIPTION>Set Dedup conf</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="dedupConfigSequence" ref="dedupConfigSequence"/>
+						<PARAM name="dedupConfigSequenceParam" value="dedup.conf.queue"/>
+						<PARAM name="entityTypeNameParam" value="entityType"/>
+						<PARAM name="entityTypeIdParam" value="entityTypeId"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="findIndex"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="findIndex" type="FindIndex">
+					<DESCRIPTION/>
+					<PARAMETERS>
+						<PARAM name="mdFormat" ref="mdFormat"/>
+						<PARAM name="layout" ref="layout"/>
+						<PARAM name="interpretation" ref="interpretation"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC name="found" to="prepareIndex"/>
+						<ARC name="notFound" to="createIndex"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="createIndex" type="CreateIndex">
+					<DESCRIPTION/>
+					<PARAMETERS>
+						<PARAM name="format" ref="mdFormat"/>
+						<PARAM name="layout" ref="layout"/>
+						<PARAM name="interpretation" ref="interpretation"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="prepareIndex"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="prepareIndex" type="PrepareDedupIndexJob">
+					<DESCRIPTION>Prepare indexing</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="mdFormat" ref="mdFormat"/>
+						<PARAM name="layout" ref="layout"/>
+						<PARAM name="interpretation" ref="interpretation"/>
+						<PARAM name="rottenRecordsPathParam" value="rottenRecordsPath"/>
+						<PARAM name="hbaseTable" ref="hbaseTable"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="cleanupRotten"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="cleanupRotten" type="DeleteHdfsPathJob">
+					<DESCRIPTION>hdfs cleanup (rotten)</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="path" env="rottenRecordsPath"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="updateIndex"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="updateIndex" type="SubmitHadoopJob">
+					<DESCRIPTION>M/O index records</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="hadoopJob" value="dedupIndexFeedJob"/>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="jobParams">
+							<MAP>
+								<ENTRY key="hbase.mapred.inputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapreduce.inputtable" ref="hbaseTable"/>
+								<ENTRY key="mapred.output.dir" env="rottenRecordsPath"/>
+								<ENTRY key="index.fields" env="index.fields"/>
+								<ENTRY key="index.solr.url" env="index.solr.url"/>
+								<ENTRY key="index.solr.collection" env="index.solr.collection"/>
+								<ENTRY key="index.buffer.flush.threshold" env="index.buffer.flush.threshold"/>
+								<ENTRY key="index.shutdown.wait.time" env="index.shutdown.wait.time"/>
+								<ENTRY key="index.solr.sim.mode" env="index.solr.sim.mode"/>
+								<ENTRY key="index.feed.timestamp" env="index.feed.timestamp"/>
+								<ENTRY key="entityTypeId" env="entityTypeId"/>
+								<ENTRY key="entityType" env="entityType"/>
+								<ENTRY key="actionset" ref="dedupConfigSequence"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="finalize"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="finalize" type="FinalizeDedupIndexFeeding">
+					<DESCRIPTION>commit changes</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="actionset" ref="dedupConfigSequence"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="updateDs"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="updateDs" type="IndexDsUpdateJob">
+					<DESCRIPTION>update DS</DESCRIPTION>
+					<PARAMETERS/>
+					<ARCS>
+						<ARC to="success"/>
+					</ARCS>
+				</NODE>
+
+
+			</WORKFLOW>
+
+		</CONFIGURATION>
+
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/update.OAI.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/update.OAI.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/update.OAI.xml	(revision 43444)
@@ -0,0 +1,135 @@
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="12f1db9b-3c3e-4fd6-a425-932afd6ef3be_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2016-06-25T10:24:32+00:00"/>
+	</HEADER>
+	<BODY>
+		<CONFIGURATION>
+
+			<PARAMETERS>
+				<PARAM name="cluster" description="Hadoop cluster name" required="true" type="string"/>
+				<PARAM name="inputRecordsPath" description="path on HDFS pointing to the sequence file with the input xml records" required="true" type="string"/>
+
+				<PARAM name="oaiDBName" description="db name" required="true" type="string"/>
+				<PARAM name="oaiCollectionName" description="collection name" required="true" type="string" default="oaf-index-openaire"/>
+				<PARAM name="skipDuplicates" description="skip duplicates" required="true" type="boolean"/>
+				<PARAM name="duplicateXPath" description="xpath to identify duplicated records" required="true" type="string" default="//*[local-name()='entity']//*[local-name()='datainfo']/*[local-name()='deletedbyinference'][./text() = 'true']"/>
+
+				<PARAM name="fieldNames" description="list of field names to index" required="true" type="string" default="set,datestamp;license,oaftype;set,oaftype;oaftype,funder;resulttypeid,oaftype;oaftype,funding;resulttypeid,funder;resulttypeid,funding"/>
+				<PARAM name="configuredOnly" description="rebuild only the indicies in the OAI configuration profile" required="true" type="boolean" default="false"/>
+			</PARAMETERS>
+
+			<WORKFLOW>
+
+				<NODE name="prepareOAI" type="PrepareOaiJob" isStart="true">
+					<DESCRIPTION>Prepare params for OAI store feeding</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="oaiConfigurationParam" value="oaiConfiguration"/>
+						<PARAM name="oaiFeedDateParam" value="oai.feed.date"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="OAIDropStore"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="OAIDropStore" type="OAIDropStore">
+					<DESCRIPTION>Drop the store: upserts are too expensive!</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="dbName" ref="oaiDBName"/>
+						<PARAM name="collectionName" ref="oaiCollectionName"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="OAICreateStore"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="OAICreateStore" type="OAICreateStore">
+					<DESCRIPTION/>
+					<PARAMETERS>
+						<PARAM name="dbName" ref="oaiDBName"/>
+						<PARAM name="collectionName" ref="oaiCollectionName"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="feedOAI"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="feedOAI" type="SubmitHadoopJob">
+					<DESCRIPTION>M/O oai feeding</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="hadoopJob" value="oaiFeedJob"/>
+						<PARAM name="jobParams">
+							<MAP>
+								<ENTRY key="mapred.input.dir" ref="inputRecordsPath"/>
+								<ENTRY key="services.publisher.oai.collection" ref="oaiCollectionName"/>
+								<ENTRY key="services.publisher.oai.db" ref="oaiDBName"/>
+								<ENTRY key="services.publisher.oai.skipDuplicates" ref="skipDuplicates"/>
+								<ENTRY key="services.publisher.oai.duplicateXPath" ref="duplicateXPath"/>
+								<ENTRY key="oaiConfiguration" env="oaiConfiguration"/>
+								<ENTRY key="oai.feed.date" env="oai.feed.date"/>
+								<ENTRY key="services.publisher.oai.host" property="services.publisher.oai.host"/>
+								<ENTRY key="services.publisher.oai.port" property="services.publisher.oai.port"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="CompoundIndexes"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="CompoundIndexes" type="OAICreateIndex">
+					<DESCRIPTION>Create composite indexes for the OAI store</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="dbName" ref="oaiDBName"/>
+						<PARAM name="collectionName" ref="oaiCollectionName"/>
+						<PARAM name="fieldNames" ref="fieldNames"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="ConfigIndexes"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="ConfigIndexes" type="OAIEnsureIndexes">
+					<DESCRIPTION>Ensure an index exists on fields as specified in the configuration profile</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="dbName" ref="oaiDBName"/>
+						<PARAM name="collectionName" ref="oaiCollectionName"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="SetsCount"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="SetsCount" type="OAISetsCountUpdate">
+					<DESCRIPTION>Count records in each OAI set, for each exported metadata format</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="dbName" ref="oaiDBName"/>
+						<PARAM name="collectionName" ref="oaiCollectionName"/>
+						<PARAM name="configuredOnly" ref="configuredOnly"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="ConfigSets"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="ConfigSets" type="RefreshSetsFromConfig">
+					<DESCRIPTION>Reads the current OAI configuration and upsert OAI sets accordingly (counts are updated as well)</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="dbName" ref="oaiDBName"/>
+						<PARAM name="collectionName" ref="oaiCollectionName"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="success"/>
+					</ARCS>
+				</NODE>
+
+			</WORKFLOW>
+
+		</CONFIGURATION>
+
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/groupEntities.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/groupEntities.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/groupEntities.xml	(revision 43444)
@@ -0,0 +1,119 @@
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="42678181-e9b4-4e2a-94cd-c6258f59552a_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2016-06-25T10:24:32+00:00"/>
+	</HEADER>
+	<BODY>
+		<CONFIGURATION>
+
+			<PARAMETERS>
+
+				<PARAM name="mdFormat" description="metadata format name" required="true" type="string"/>
+				<PARAM name="layout" description="metadata layout name" required="true" type="string"/>
+				<PARAM name="interpretation" description="metadata interpretation name" required="true" type="string"/>
+
+				<PARAM name="hbaseTable" description="HBase table name" required="true" type="string"/>
+				<PARAM name="cluster" description="Hadoop cluster name" required="true" type="string"/>
+
+				<PARAM name="index.entity.links" description="Hadoop cluster name" required="true" type="string"/>
+				<PARAM name="contextmap" description="Hadoop cluster name" required="true" type="string"/>
+				<PARAM name="relClasses" description="Hadoop cluster name" required="true" type="string"/>
+
+			</PARAMETERS>
+
+			<WORKFLOW>
+
+				<!-- UPDATE INDEX -->
+				<NODE name="findIndex" type="FindIndex" isStart="true">
+					<DESCRIPTION/>
+					<PARAMETERS>
+						<PARAM name="mdFormat" ref="mdFormat"/>
+						<PARAM name="layout" ref="layout"/>
+						<PARAM name="interpretation" ref="interpretation"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC name="found" to="prepareIndexing"/>
+						<ARC name="notFound" to="createIndex"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="createIndex" type="CreateIndex">
+					<DESCRIPTION/>
+					<PARAMETERS>
+						<PARAM name="format" ref="mdFormat"/>
+						<PARAM name="layout" ref="layout"/>
+						<PARAM name="interpretation" ref="interpretation"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="prepareIndexing"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="prepareIndexing" type="PrepareIndexJob">
+					<DESCRIPTION>Prepare indexing</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="outputRecordsPathParam" value="hdfsRecordsPath"/>
+						<PARAM name="rottenRecordsPathParam" value="rottenRecordsPath"/>
+						<PARAM name="layoutToRecordStylesheet" value="/eu/dnetlib/msro/workflows/index//openaireLayoutToRecordStylesheet.xsl"/>
+						<PARAM name="oafSchemaLocation" property="oaf.schema.location"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="cleanupXml"/>
+						<ARC to="cleanupRotten"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="cleanupXml" type="DeleteHdfsPathJob">
+					<DESCRIPTION>hdfs cleanup (xml)</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="path" env="hdfsRecordsPath"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="groupEntities"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="cleanupRotten" type="DeleteHdfsPathJob">
+					<DESCRIPTION>hdfs cleanup (rotten)</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="path" env="rottenRecordsPath"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="groupEntities"/>
+					</ARCS>
+				</NODE>
+
+
+				<NODE isJoin="true" name="groupEntities" type="SubmitHadoopJob">
+					<DESCRIPTION>redirect rels</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="hadoopJob" value="prepareIndexDataJob"/>
+						<PARAM name="jobParams">
+							<MAP>
+								<ENTRY key="hbase.mapred.inputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapreduce.inputtable" ref="hbaseTable"/>
+								<ENTRY key="mapred.output.dir" env="hdfsRecordsPath"/>
+								<ENTRY key="index.entity.links" ref="index.entity.links"/>
+								<ENTRY key="contextmap" ref="contextmap"/>
+								<ENTRY key="relClasses" ref="relClasses"/>
+								<ENTRY key="oaf.schema.location" property="oaf.schema.location"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="success"/>
+					</ARCS>
+				</NODE>
+
+			</WORKFLOW>
+
+		</CONFIGURATION>
+
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/dedup.global.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/dedup.global.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/dedup.global.xml	(revision 43444)
@@ -0,0 +1,176 @@
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="80f76a00-0eb3-4df0-a256-99f1df7b5fd8_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl"/>
+		<RESOURCE_TYPE value="WorkflowDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2006-05-04T18:13:51.0Z"/>
+	</HEADER>
+	<BODY>
+		<WORKFLOW_NAME menuSection="InfoSpace Deduplication">InfoSpace Deduplication</WORKFLOW_NAME>
+		<WORKFLOW_DESCRIPTION>OpenAIRE Deduplication</WORKFLOW_DESCRIPTION>
+		<WORKFLOW_INFO/>
+		<WORKFLOW_FAMILY>InfoSpace Deduplication</WORKFLOW_FAMILY>
+		<WORKFLOW_PRIORITY>35</WORKFLOW_PRIORITY>
+		<!-- <ADMIN_EMAIL>alessia.bardi@isti.cnr.it,claudio.atzori@isti.cnr.it</ADMIN_EMAIL> -->
+		<CONFIGURATION status="EXECUTABLE" start="MANUAL">
+
+			<PARAMETERS>
+				<PARAM name="table" description="HBase table to be used in this workflow" required="true" managedBy="user" type="string"></PARAM>
+				<PARAM name="cluster" description="Hadoop cluster logical name" required="true" managedBy="user" type="string" function="validValues(['DM','IIS'])"></PARAM>
+				<PARAM name="reuseRegionInfo" description="Reuse table splits" required="true" managedBy="user" type="boolean">true</PARAM>
+
+				<PARAM name="tableConf" description="HBase table configuration param name" required="true" managedBy="system" type="string">hbase.table.conf</PARAM>
+				<PARAM name="tableCols" description="HBase table schema param name" required="true" managedBy="system" type="string">hbase.table.cols</PARAM>
+
+				<PARAM name="db" description="relational db name" required="true" type="string" managedBy="user"/>
+
+				<PARAM name="sqlQueryORG" description="the SQL query" required="true" managedBy="system" type="string">/eu/dnetlib/msro/workflows/hbase/queryOrganizations.sql</PARAM>
+				<PARAM name="mappingORG" description="xslt mapping function" required="true" type="string" managedBy="user" function="obtainValues('dbmf2hbaseMappings', {})"/>
+
+				<PARAM name="webcrawlActionSet" description="action set name for WebCrawl" required="true" type="string" managedBy="system">iis-wos-entities</PARAM>
+
+				<PARAM name="mappingPublication" description="xslt mapping for publications" required="true" type="string" managedBy="user" function="obtainValues('oaf2hbaseMappings', {})"/>
+				<PARAM name="reusePublication" description="reuse publications on HDFS?" required="true" type="boolean" managedBy="user"/>
+				<PARAM name="hdfsPathPublication" description="hdfs path for publications" required="true" type="string" managedBy="user"/>
+
+				<PARAM name="dedupConfigSequence" description="dedup configuration orchestration name" required="true" type="string" function="obtainValues('dedupOrchestrations', {})" managedBy="user"/>
+				<PARAM name="minDistWorkDir" description="work directory for the minDist algorithm" required="true" type="string" managedBy="user">/tmp/dedup/prod/mindist</PARAM>
+
+				<PARAM name="mappingSimilarities" description="xslt mapping function for similarities" required="true" type="string" managedBy="user" function="obtainValues('dbmf2hbaseMappings', {})"/>
+				<PARAM name="mappingDissimilarities" description="xslt mapping function for dissimilarities" required="true" type="string" managedBy="user" function="obtainValues('dbmf2hbaseMappings', {})"/>
+			</PARAMETERS>
+			<WORKFLOW>
+
+				<NODE name="resetHbase" type="LaunchWorkflowTemplate" isStart="true">
+					<DESCRIPTION>Reset HBase table</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="75345aba-c069-43f4-90aa-e13688d9845e_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="table" ref="table"/>
+								<ENTRY key="cluster" ref="cluster"/>
+								<ENTRY key="reuseRegionInfo" ref="reuseRegionInfo"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+
+					<ARCS>
+						<ARC to="select"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="select" type="SelectWfPathByDedupEntity">
+					<DESCRIPTION>Select the workflow path</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="dedupConfigSequence" ref="dedupConfigSequence"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC name="organization" to="orgs2hbase"/>
+						<ARC name="result" to="webcrawl2hbase"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="orgs2hbase" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>Organizations to HBase</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="9bb04513-be1d-4d13-8e86-785cc0375635_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="sqlQuery" ref="sqlQueryORG"/>
+								<ENTRY key="mapping" ref="mappingORG"/>
+								<ENTRY key="sqlPrepareQuery" value="/eu/dnetlib/msro/workflows/hbase/prepareQueryDefault.sql"/>
+								<ENTRY key="db" ref="db"/>
+								<ENTRY key="hbaseTable" ref="table"/>
+								<ENTRY key="cluster" ref="cluster"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="deduplication"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="webcrawl2hbase" type="PromoteActionsHDFS">
+					<DESCRIPTION>Promote Actions</DESCRIPTION>
+					<PARAMETERS>
+						<!-- dedup-similarity-organization-simple,dedup-similarity-result-levenstein,iis-dataset-entities-main,iis-referenced-datasets-preprocessing,iis-dataset-entities-preprocessing,iis-document-citations,iis-document-classes,iis-referenced-projects-main,iis-document-similarities,iis-document-statistics,iis-extracted-metadata,iis-referenced-datasets-main,iis-referenced-projects-preprocessing,iis-researchinitiative,iis-wos-entities,iis-referenceextraction-pdb -->
+						<PARAM name="set" ref="webcrawlActionSet"/>
+						<PARAM name="tableName" ref="table"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="oaf2hbase"/>
+					</ARCS>
+				</NODE>
+
+
+				<NODE name="oaf2hbase" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>OAF Publications to HBase</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="5b05a65a-4eeb-4862-bc55-b35c7ec3baf0_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="hbaseTable" ref="table"/>
+								<ENTRY key="cluster" ref="cluster"/>
+								<ENTRY key="mdFormat" value="OAF"/>
+								<ENTRY key="mapping" ref="mappingPublication"/>
+								<ENTRY key="reuseMdRecords" ref="reusePublication"/>
+								<ENTRY key="hdfsPath" ref="hdfsPathPublication"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="deduplication"/>
+					</ARCS>
+				</NODE>
+
+
+				<NODE name="deduplication" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>Deduplication</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="01ed11e8-e874-4478-a8ac-83e63e9699e4_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="hbaseTable" ref="table"/>
+								<ENTRY key="cluster" ref="cluster"/>
+								<ENTRY key="dedupConfigSequence" ref="dedupConfigSequence"/>
+								<ENTRY key="minDistWorkDir" ref="minDistWorkDir"/>
+								<ENTRY key="mappingSimilarities" ref="mappingSimilarities"/>
+								<ENTRY key="mappingDissimilarities" ref="mappingDissimilarities"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="index"/>
+					</ARCS>
+				</NODE>
+
+
+				<NODE name="index" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>Update dedup index</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="a1b0b30c-7e19-46e2-888c-67c0b880b346_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="hbaseTable" ref="table"/>
+								<ENTRY key="cluster" ref="cluster"/>
+								<ENTRY key="dedupConfigSequence" ref="dedupConfigSequence"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="success"/>
+					</ARCS>
+				</NODE>
+
+
+			</WORKFLOW>
+		</CONFIGURATION>
+		<NOTIFICATIONS/>
+		<SCHEDULING enabled="false">
+			<CRON>29 5 22 ? * *</CRON>
+			<MININTERVAL>10080</MININTERVAL>
+		</SCHEDULING>
+		<STATUS/>
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/reset.hbase.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/reset.hbase.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/reset.hbase.xml	(revision 43444)
@@ -0,0 +1,75 @@
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="75345aba-c069-43f4-90aa-e13688d9845e_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2006-05-04T18:13:51.0Z"/>
+	</HEADER>
+	<BODY>
+		<CONFIGURATION>
+
+			<PARAMETERS>
+				<PARAM name="table" description="HBase table to be dropped" required="true" type="string"/>
+				<PARAM name="cluster" description="Hadoop cluster logical name" required="true" type="string"/>
+				<PARAM name="reuseRegionInfo" description="Reuse table splits" required="true" type="boolean"/>
+
+				<PARAM name="tableConf" description="HBase table configuration param name" type="string" default="hbase.table.conf"/>
+				<PARAM name="tableCols" description="HBase table schema param name" type="string" default="hbase.table.cols"/>
+			</PARAMETERS>
+
+			<WORKFLOW>
+
+				<NODE name="checkTable" type="CheckHBaseTable" isStart="true">
+					<DESCRIPTION>check hbase table</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="tableName" ref="table"/>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="tableConfigurationParamName" ref="tableConf"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="drop" name="drop"/>
+						<ARC to="define" name="define"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="drop" type="DropHBaseTable">
+					<DESCRIPTION>drop hbase table</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="tableName" ref="table"/>
+						<PARAM name="cluster" ref="cluster"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="define"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="define" type="DefineHBaseOpenaireSchema">
+					<DESCRIPTION>define OpenAIRE hbase table</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="tableColumnsParamName" ref="tableCols"/>
+						<PARAM name="tableName" ref="table"/>
+						<PARAM name="cluster" ref="cluster"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="create"/>
+					</ARCS>
+				</NODE>
+				<NODE name="create" type="CreateHBaseTable">
+					<DESCRIPTION>create hbase table</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="tableName" ref="table"/>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="tableColumnsParamName" ref="tableCols"/>
+						<PARAM name="tableConfigurationParamName" ref="tableConf"/>
+						<PARAM name="reuseRegionInfo" ref="reuseRegionInfo"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="success"/>
+					</ARCS>
+				</NODE>
+
+			</WORKFLOW>
+		</CONFIGURATION>
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/entityRegistry/wds_transform_er_template.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/entityRegistry/wds_transform_er_template.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/entityRegistry/wds_transform_er_template.xml	(revision 43444)
@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="276fa777-49eb-43f5-bf6f-c77751614f80_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2001-12-31T12:00:00"/>
+	</HEADER>
+	<BODY>
+		<CONFIGURATION>
+			<PARAMETERS>
+				<PARAM name="dsId" description="Datasource Id" required="true" type="string"/>
+				<PARAM name="interface" description="Datasource Interface" required="true" type="string"/>
+				<PARAM name="cleanMdstoreId" description="Store for cleaned records" required="true" type="string"/>
+				<PARAM name="collMdstoreId" description="Store for collected records" required="true" type="string"/>
+				<PARAM name="transformRuleId" description="Transformation Rule" required="true" type="string"/>
+				<PARAM name="cleanRuleId" description="Cleaning Rule" required="true" type="string"/>
+			</PARAMETERS>
+			<WORKFLOW>
+				<NODE name="obtainParams" type="ObtainOpenaireDataSourceParams" isStart="true">
+					<DESCRIPTION>Fetch records from MDStore</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="providerId" ref="dsId"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="fetchOriginals"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="fetchOriginals" type="FetchMDStoreRecords">
+					<DESCRIPTION>Fetch records from MDStore</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="mdId" ref="collMdstoreId"/>
+						<PARAM name="mdFormat" value="GMF"/>
+						<PARAM name="eprParam" value="orig_epr"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="transform"/>
+					</ARCS>
+				</NODE>
+				<NODE name="transform" type="ApplyXslt">
+					<DESCRIPTION>Transform to DB records</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="xsltClasspath" value="/eu/dnetlib/wds/workflows/repo-hi/xslt/cmr_er2db.xslt"/>
+						<PARAM name="inputEprParam" value="orig_epr"/>
+						<PARAM name="outputEprParam" value="db_epr"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="clean"/>
+					</ARCS>
+				</NODE>
+				<NODE name="clean" type="Clean">
+					<DESCRIPTION>Clean DB record</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="ruleId" ref="cleanRuleId"/>
+						<PARAM name="inputEprParam" value="db_epr"/>
+						<PARAM name="outputEprParam" value="clean_epr"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="storeRecords"/>
+					</ARCS>
+				</NODE>
+				<NODE name="storeRecords" type="StoreMDStoreRecords">
+					<DESCRIPTION>Store mdtore records</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="mdId" ref="cleanMdstoreId"/>
+						<PARAM name="storingType" value="REFRESH"/>
+						<PARAM name="eprParam" value="clean_epr"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="UPDATE_INFO"/>
+					</ARCS>
+				</NODE>
+				<NODE name="UPDATE_INFO" type="MDStoreToApiExtraField">
+					<DESCRIPTION>Update datasouce API extra fields</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="mdId" ref="cleanMdstoreId"/>
+						<PARAM name="datasourceId" ref="dsId"/>
+						<PARAM name="datasourceInterface" ref="interface"/>
+						<PARAM name="extraFieldForTotal" value="last_aggregation_total"/>
+						<PARAM name="extraFieldForDate" value="last_aggregation_date"/>
+						<PARAM name="extraFieldForMdId" value="last_aggregation_mdId"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="success"/>
+					</ARCS>
+				</NODE>
+			</WORKFLOW>
+		</CONFIGURATION>
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/update.provision.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/update.provision.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/update.provision.xml	(revision 43444)
@@ -0,0 +1,255 @@
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="a4434d62-d4cd-4c73-a107-bc7c62e6f815_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2016-06-25T10:24:32+00:00"/>
+	</HEADER>
+	<BODY>
+		<CONFIGURATION>
+
+			<PARAMETERS>
+
+				<PARAM name="mdFormat" description="metadata format name" required="true" type="string"/>
+				<PARAM name="layout" description="metadata format name" required="true" type="string" default="index"/>
+				<PARAM name="interpretation" description="metadata format name" required="true" type="string" default="openaire"/>
+
+				<PARAM name="hbaseTable" description="HBase table name" required="true" type="string"/>
+				<PARAM name="cluster" description="Hadoop cluster name" required="true" type="string"/>
+
+				<PARAM name="oaiDBName" description="OAI store db name" required="true" type="string"/>
+
+			</PARAMETERS>
+
+			<WORKFLOW>
+
+				<!-- PREPARE NODES -->
+				<!--
+				<NODE isStart="true" name="setInfo" type="SetFormatInfo">
+					<DESCRIPTION>set mdformat, layout, interpretation</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="format" ref="mdFormat" />
+						<PARAM name="layout" ref="index" />
+						<PARAM name="interpretation" ref="openaire" />
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="groupEntities"/>
+					</ARCS>
+				</NODE>
+				-->
+
+				<NODE isStart="true" name="fetchRelClasses" type="FetchRelClasses">
+					<DESCRIPTION>loads the link semantic profiles</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="xquery" property="dnet.openaire.model.relclasses.xquery"/>
+						<PARAM name="relClassesName" value="relClasses"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="fetchContexts"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="fetchContexts" type="LoadContextsJob">
+					<DESCRIPTION>loads the context profiles</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="contextParam" value="contextmap"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="fetchEntityLinks"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="fetchEntityLinks" type="LoadEntityLinksJob">
+					<DESCRIPTION>loads the entity link profile</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="entityLinksParam" value="index.entity.links"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="groupEntities"/>
+					</ARCS>
+				</NODE>
+
+
+				<!-- UPDATE INDEX -->
+				<NODE name="findIndex" type="FindIndex" isStart="true">
+					<DESCRIPTION>Search for an index DS</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="mdFormat" ref="mdFormat"/>
+						<PARAM name="layout" ref="layout"/>
+						<PARAM name="interpretation" ref="interpretation"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC name="found" to="prepareIndexing"/>
+						<ARC name="notFound" to="createIndex"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="createIndex" type="CreateIndex">
+					<DESCRIPTION>creates the index</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="format" ref="mdFormat"/>
+						<PARAM name="layout" ref="layout"/>
+						<PARAM name="interpretation" ref="interpretation"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="prepareIndexing"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="prepareIndexing" type="PrepareIndexJob">
+					<DESCRIPTION>Prepare parameters for indexing</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="hbaseTable" ref="hbaseTable"/>
+						<PARAM name="outputRecordsPathParam" value="hdfsRecordsPath"/>
+						<PARAM name="rottenRecordsPathParam" value="rottenRecordsPath"/>
+						<PARAM name="layoutToRecordStylesheet" value="/eu/dnetlib/msro/workflows/index//openaireLayoutToRecordStylesheet.xsl"/>
+						<PARAM name="oafSchemaLocation" property="oaf.schema.location"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="cleanupXml"/>
+						<ARC to="cleanupRotten"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="cleanupXml" type="DeleteHdfsPathJob">
+					<DESCRIPTION>hdfs cleanup (xml)</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="path" env="hdfsRecordsPath"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="groupEntities"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="cleanupRotten" type="DeleteHdfsPathJob">
+					<DESCRIPTION>hdfs cleanup (rotten)</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="path" env="rottenRecordsPath"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="groupEntities"/>
+					</ARCS>
+				</NODE>
+
+
+				<NODE isJoin="true" name="groupEntities" type="SubmitHadoopJob">
+					<DESCRIPTION>join the entities</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="hadoopJob" value="prepareIndexDataJob"/>
+						<PARAM name="jobParams">
+							<MAP>
+								<ENTRY key="hbase.mapred.inputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.mapreduce.inputtable" ref="hbaseTable"/>
+								<ENTRY key="mapred.output.dir" env="hdfsRecordsPath"/>
+								<ENTRY key="index.entity.links" env="index.entity.links"/>
+								<ENTRY key="contextmap" env="contextmap"/>
+								<ENTRY key="relClasses" env="relClasses"/>
+								<ENTRY key="oaf.schema.location" property="oaf.schema.location"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="updateIndex"/>
+						<!--<ARC to="updateStats"/>-->
+						<ARC to="updateOAI"/>
+					</ARCS>
+				</NODE>
+
+
+				<NODE name="updateIndex" type="SubmitHadoopJob">
+					<DESCRIPTION>feeds the index</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="hadoopJob" value="indexFeedJob"/>
+						<PARAM name="jobParams">
+							<MAP>
+								<ENTRY key="mapred.input.dir" env="hdfsRecordsPath"/>
+								<ENTRY key="mapred.output.dir" env="rottenRecordsPath"/>
+								<ENTRY key="index.xslt" env="index.xslt"/>
+								<ENTRY key="index.solr.url" env="index.solr.url"/>
+								<ENTRY key="index.solr.collection" env="index.solr.collection"/>
+								<ENTRY key="index.buffer.flush.threshold" env="index.buffer.flush.threshold"/>
+								<ENTRY key="index.shutdown.wait.time" env="index.shutdown.wait.time"/>
+								<ENTRY key="index.solr.sim.mode" env="index.solr.sim.mode"/>
+								<ENTRY key="index.feed.timestamp" env="index.feed.timestamp"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="finalize"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="finalize" type="FinalizeIndexFeeding">
+					<DESCRIPTION>commit changes</DESCRIPTION>
+					<PARAMETERS/>
+					<ARCS>
+						<ARC to="updateDs"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="updateDs" type="IndexDsUpdateJob">
+					<DESCRIPTION>update DS</DESCRIPTION>
+					<PARAMETERS/>
+					<ARCS>
+						<ARC to="waitAll"/>
+					</ARCS>
+				</NODE>
+
+				<!--  UPDATE STATS -->
+				<!--
+				<NODE name="updateStats" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>update stats</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="b921c8fc-00e8-48c3-901a-5343e339d9fe_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ==" />
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="cluster"                    ref="cluster" />
+								<ENTRY key="Stats_Hbase_Source_Table"   ref="hbaseTable"/>
+								<ENTRY key="Stats_indexConf"            env="index.entity.links"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="waitAll"/>
+					</ARCS>
+				</NODE>
+				-->
+
+				<!-- OAI STORE UPDATE -->
+				<NODE name="updateOAI" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>update OAI store</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="12f1db9b-3c3e-4fd6-a425-932afd6ef3be_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="cluster" ref="cluster"/>
+								<ENTRY key="inputRecordsPath" env="hdfsRecordsPath"/>
+								<ENTRY key="oaiDBName" ref="oaiDBName"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="waitAll"/>
+					</ARCS>
+				</NODE>
+
+				<!-- WAIT FOR ALL THE WF BRANCHES TO COMPLETE -->
+				<NODE isJoin="true" name="waitAll">
+					<DESCRIPTION>wait for all the branches to complete</DESCRIPTION>
+					<PARAMETERS/>
+					<ARCS>
+						<ARC to="success"/>
+					</ARCS>
+				</NODE>
+
+			</WORKFLOW>
+
+		</CONFIGURATION>
+
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_hi_datasource_wds.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_hi_datasource_wds.xml	(revision 43443)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_hi_datasource_wds.xml	(revision 43444)
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <RESOURCE_PROFILE>
 	<HEADER>
-		<RESOURCE_IDENTIFIER value="507b7858-a02c-4354-a3b5-13d42f1e7a26_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl"/>
+		<RESOURCE_IDENTIFIER value="d11c21eb-acd4-4d51-b943-42ff73c5482b_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl"/>
 		<RESOURCE_TYPE value="WorkflowDSResourceType"/>
 		<RESOURCE_KIND value="WorkflowDSResources"/>
 		<RESOURCE_URI value=""/>
@@ -24,8 +24,7 @@
 				<NODE name="VerifyDatasource" type="VerifyDatasource" isStart="true">
 					<DESCRIPTION>Verify if DS is pending</DESCRIPTION>
 					<PARAMETERS>
-						<PARAM name="expectedInterfaceTypologyPrefixes" value="wds"/>
-						<PARAM name="expectedCompliancePrefixes" value="entityregistry"/>
+						<PARAM name="expectedInterfaceTypologyPrefixes" value="entityregistry"/>
 					</PARAMETERS>
 					<ARCS>
 						<ARC to="registerWf"/>
@@ -58,7 +57,7 @@
 					<DESCRIPTION>Create oai_dc store</DESCRIPTION>
 					<PARAMETERS>
 						<PARAM name="format" value="GMF"/>
-						<PARAM name="interpretation" value="native"/>
+						<PARAM name="interpretation" value="datasource_native"/>
 						<PARAM name="layout" value="store"/>
 						<PARAM name="outputPrefix" value="harv_"/>
 					</PARAMETERS>
@@ -80,7 +79,6 @@
 					</ARCS>
 				</NODE>
 
-
 				<NODE name="updateParameters" type="UpdateWfParameters" isJoin="true">
 					<DESCRIPTION>Update Workflow Parameters</DESCRIPTION>
 					<PARAMETERS>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/wds/collector/plugins/applicationContext-wds-collector-plugins.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/wds/collector/plugins/applicationContext-wds-collector-plugins.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/wds/collector/plugins/applicationContext-wds-collector-plugins.xml	(revision 43444)
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:p="http://www.springframework.org/schema/p"
+       xmlns:http="http://cxf.apache.org/transports/http/configuration"
+       xmlns="http://www.springframework.org/schema/beans"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+
+
+                                    http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd">
+
+
+	<bean id="cMRCollectorPlugin" class="eu.dnetlib.wds.collector.plugins.CMRCollectorPlugin"
+	      p:classPathTemplate="classpath:/eu/dnetlib/wds/collector/plugins/CMR2Datacite.st">
+		<property name="protocolDescriptor">
+			<bean class="eu.dnetlib.rmi.data.ProtocolDescriptor" p:name="CMRCollectorPlugin"/>
+		</property>
+	</bean>
+
+
+	<bean id="CMREntityRegistryPlugin" class="eu.dnetlib.wds.collector.plugins.CMRDatasourcePlugin"
+	      p:xmlTemplate-ref="CMRDsTemplate">
+		<property name="protocolDescriptor">
+			<bean class="eu.dnetlib.rmi.data.ProtocolDescriptor" p:name="CMREntityRegistry"/>
+		</property>
+	</bean>
+
+
+	<bean id="CMRDsTemplate"
+	      class="eu.dnetlib.springutils.stringtemplate.StringTemplateFactory"
+	      p:template="classpath:/eu/dnetlib/wds/collector/plugins/CMR2Datasource.st"
+	      scope="prototype"/>
+
+
+</beans>
Index: modules/dnet-modular-uis/trunk/src/main/java/eu/dnetlib/functionality/modular/ui/workflows/controllers/WorkflowsController.java
===================================================================
--- modules/dnet-modular-uis/trunk/src/main/java/eu/dnetlib/functionality/modular/ui/workflows/controllers/WorkflowsController.java	(revision 43443)
+++ modules/dnet-modular-uis/trunk/src/main/java/eu/dnetlib/functionality/modular/ui/workflows/controllers/WorkflowsController.java	(revision 43444)
@@ -5,57 +5,23 @@
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.Spliterator;
-import java.util.Spliterators;
+import java.util.*;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
-
 import javax.annotation.Resource;
 import javax.servlet.http.HttpServletResponse;
 
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.math.NumberUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.dom4j.Document;
-import org.dom4j.Element;
-import org.dom4j.Node;
-import org.dom4j.io.SAXReader;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.ResponseBody;
-
 import com.google.common.base.Splitter;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
-
 import eu.dnetlib.enabling.locators.UniqueServiceLocator;
 import eu.dnetlib.functionality.modular.ui.AbstractAjaxController;
 import eu.dnetlib.functionality.modular.ui.repositories.objects.RepoHIWorkflow;
 import eu.dnetlib.functionality.modular.ui.repositories.objects.VocabularyEntry;
 import eu.dnetlib.functionality.modular.ui.repositories.util.RepoUIUtils;
-import eu.dnetlib.functionality.modular.ui.workflows.objects.EnvParam;
-import eu.dnetlib.functionality.modular.ui.workflows.objects.GraphDetails;
-import eu.dnetlib.functionality.modular.ui.workflows.objects.ProcessInfo;
-import eu.dnetlib.functionality.modular.ui.workflows.objects.WorkflowItem;
-import eu.dnetlib.functionality.modular.ui.workflows.objects.WorkflowNotificationInfo;
-import eu.dnetlib.functionality.modular.ui.workflows.objects.WorkflowUpdateInfo;
+import eu.dnetlib.functionality.modular.ui.workflows.objects.*;
 import eu.dnetlib.functionality.modular.ui.workflows.objects.sections.WorkflowSectionGrouper;
 import eu.dnetlib.functionality.modular.ui.workflows.util.ISLookupClient;
 import eu.dnetlib.functionality.modular.ui.workflows.util.ISRegistryClient;
@@ -78,6 +44,22 @@
 import eu.dnetlib.rmi.enabling.ISLookUpException;
 import eu.dnetlib.rmi.enabling.ISRegistryException;
 import eu.dnetlib.rmi.manager.MSROException;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.Element;
+import org.dom4j.Node;
+import org.dom4j.io.SAXReader;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseBody;
 
 // import com.google.common.collect.Lists;
 
@@ -228,6 +210,9 @@
 		final Document doc = reader.read(new StringReader(profile));
 
 		final Node ifcNode = doc.selectSingleNode("//INTERFACE[@id = '" + ifaceId + "']");
+
+		final String profileId = doc.valueOf("//RESOURCE_IDENTIFIER/@value");
+
 		final String protocol = ifcNode.valueOf("./ACCESS_PROTOCOL");
 		final Element extra = doc.getRootElement().addElement("extra_info");
 
@@ -252,7 +237,7 @@
 		}
 
 		final Element wfs = extra.addElement("workflows");
-		for (final WorkflowItem item : this.isLookupClient.listWorflowsForApi(repoId, ifaceId)) {
+		for (final WorkflowItem item : this.isLookupClient.listWorflowsForApi(profileId, ifaceId)) {
 			final Element wf = wfs.addElement("workflow");
 			wf.addAttribute("id", item.getWfId());
 			wf.addAttribute("name", item.getName());
@@ -261,7 +246,7 @@
 				wf.addAttribute("destroy", "1");
 			}
 		}
-		final Map<String, String> params = new HashMap<String, String>();
+		final Map<String, String> params = new HashMap<>();
 		params.put("profileId", doc.valueOf("//RESOURCE_IDENTIFIER/@value"));
 		params.put("ifaceId", ifaceId);
 		params.put("protocol", protocol);
@@ -504,7 +489,7 @@
 				try {
 					WorkflowsController.this.serviceLocator.getService(DatasourceManagerService.class).updateActivationStatus(repoId, ifaceId, true);
 				} catch (final DatasourceManagerServiceException e) {
-					log.error("Error updating activation status of " + repoId);
+					log.error("Error updating activation status of " + repoId, e);
 				}
 			}
 
Index: modules/dnet-msro-service/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/misc/FindVocabularyEntriesJobNode.java
===================================================================
--- modules/dnet-msro-service/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/misc/FindVocabularyEntriesJobNode.java	(nonexistent)
+++ modules/dnet-msro-service/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/misc/FindVocabularyEntriesJobNode.java	(revision 43444)
@@ -0,0 +1,55 @@
+package eu.dnetlib.msro.workflows.nodes.misc;
+
+import java.util.List;
+
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.resultset.factory.ResultSetFactory;
+import eu.dnetlib.msro.workflows.graph.Arc;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.procs.Env;
+import eu.dnetlib.rmi.common.ResultSet;
+import eu.dnetlib.rmi.enabling.ISLookUpService;
+import org.apache.commons.io.IOUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Required;
+import org.springframework.core.io.Resource;
+
+public class FindVocabularyEntriesJobNode extends SimpleJobNode {
+
+	private String outputEprParam = "epr";
+
+	@javax.annotation.Resource
+	private UniqueServiceLocator serviceLocator;
+
+	@Autowired
+	private ResultSetFactory resultSetFactory;
+
+	private Resource xquery;
+
+	@Override
+	protected String execute(final Env env) throws Exception {
+		final String query = IOUtils.toString(xquery.getInputStream());
+		final List<String> terms = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(query);
+		final ResultSet<String> resultSet = resultSetFactory.createResultSet(terms);
+		env.setAttribute(getOutputEprParam(), resultSet);
+		return Arc.DEFAULT_ARC;
+	}
+
+	public String getOutputEprParam() {
+		return outputEprParam;
+	}
+
+	public void setOutputEprParam(final String outputEprParam) {
+		this.outputEprParam = outputEprParam;
+	}
+
+	public Resource getXquery() {
+		return xquery;
+	}
+
+	@Required
+	public void setXquery(final Resource xquery) {
+		this.xquery = xquery;
+	}
+
+}
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/wds/workflows/repo-hi/xslt/cmr_er2db.xslt
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/wds/workflows/repo-hi/xslt/cmr_er2db.xslt	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/wds/workflows/repo-hi/xslt/cmr_er2db.xslt	(revision 43444)
@@ -0,0 +1,42 @@
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                xmlns:xs="http://www.w3.org/2001/XMLSchema"
+                exclude-result-prefixes="xs"
+                version="2.0">
+	<xsl:param name="parentDatasourceId"/>
+	<xsl:param name="namespacePrefix"/>
+
+	<xsl:template match="/">
+		<record>
+			<xsl:copy-of select=".//*[local-name()='header']"/>
+			<metadata>
+				<ROWS>
+					<ROW table="datasources">
+						<FIELD name="id">
+							<xsl:value-of select="concat($namespacePrefix,'::', //datasource/id)"/>
+						</FIELD>
+						<FIELD name="_dnet_resource_identifier_">
+							<xsl:value-of select="concat($namespacePrefix,'::', //datasource/id)"/>
+						</FIELD>
+						<FIELD name="officialname">
+							<xsl:value-of select="//datasource/name"/>
+						</FIELD>
+						<FIELD name="englishname">
+							<xsl:value-of select="//datasource/name"/>
+						</FIELD>
+						<FIELD name="od_contenttypes">dataset</FIELD>
+						<FIELD name="websiteurl">
+							<xsl:value-of select="//datasource/url"/>
+						</FIELD>
+						<FIELD name="namespaceprefix">
+							<xsl:value-of select="concat('cmr_',substring(//datasource/uuid,1,8))"/>
+						</FIELD>
+						<FIELD name="datasourceclass">datarepository</FIELD>
+						<FIELD name="collectedfrom">
+							<xsl:value-of select="$parentDatasourceId"/>
+						</FIELD>
+					</ROW>
+				</ROWS>
+			</metadata>
+		</record>
+	</xsl:template>
+</xsl:stylesheet>
\ No newline at end of file
Index: modules/dnet-data-support-services/trunk/src/main/java/eu/dnetlib/enabling/datasources/DatasourceFunctions.java
===================================================================
--- modules/dnet-data-support-services/trunk/src/main/java/eu/dnetlib/enabling/datasources/DatasourceFunctions.java	(revision 43443)
+++ modules/dnet-data-support-services/trunk/src/main/java/eu/dnetlib/enabling/datasources/DatasourceFunctions.java	(revision 43444)
@@ -115,7 +115,7 @@
 	}
 
 	public static String asSqlValue(final String s) {
-		return s == null ? "NULL" : "'" + s + "'";
+		return s == null ? "NULL" : "'" + StringUtils.replace(s, "'", "''") + "'";
 	}
 
 	public static String asSqlValue(final Boolean b) {
Index: modules/dnet-msro-service/trunk/src/main/resources/eu/dnetlib/msro/workflows/xqueries/vocabulary_entries.xquery
===================================================================
--- modules/dnet-msro-service/trunk/src/main/resources/eu/dnetlib/msro/workflows/xqueries/vocabulary_entries.xquery	(nonexistent)
+++ modules/dnet-msro-service/trunk/src/main/resources/eu/dnetlib/msro/workflows/xqueries/vocabulary_entries.xquery	(revision 43444)
@@ -0,0 +1,13 @@
+for $x in collection("/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType")//TERM
+where
+    string-length($x/@english_name) > 0 and string-length($x/@code) > 0
+return
+    <entry>
+        <termEncoding>{$x/@encoding/string()}</termEncoding>
+        <termCode>{$x/@code/string()}</termCode>
+        <termEnglishName>{$x/@english_name/string()}</termEnglishName>
+        <termNativeName>{$x/@native_name/string()}</termNativeName>
+        <vocCode>{$x/../../VOCABULARY_NAME/@code/string()}</vocCode>
+        <vocName>{$x/../../VOCABULARY_NAME/text()}</vocName>
+        <vocDescription>{$x/../../VOCABULARY_DESCRIPTION/text()}</vocDescription>
+    </entry>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/updateStats.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/updateStats.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/updateStats.xml	(revision 43444)
@@ -0,0 +1,56 @@
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="b921c8fc-00e8-48c3-901a-5343e339d9fe_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl"/>
+		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2016-06-25T10:24:32+00:00"/>
+	</HEADER>
+	<BODY>
+		<CONFIGURATION>
+
+			<PARAMETERS>
+
+				<PARAM name="hbaseTable" description="HBase table name" required="true" type="string"/>
+				<PARAM name="cluster" description="Hadoop cluster name" required="true" type="string"/>
+				<PARAM name="index.entity.links" description="descriptor for relationships among entities" required="true" type="string"/>
+
+			</PARAMETERS>
+
+			<WORKFLOW>
+
+				<NODE name="prepareStats" type="PrepareStatsParams" isStart="true">
+					<DESCRIPTION>set params for stats</DESCRIPTION>
+					<PARAMETERS/>
+					<ARCS>
+						<ARC to="updateStats"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="updateStats" type="SubmitHadoopJob">
+					<DESCRIPTION>Update stats DB</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="hadoopJob" value="StatsExportJob"/>
+						<PARAM name="simulation" value="false"/>
+						<PARAM name="jobParams">
+							<MAP>
+								<ENTRY key="Stats_Hbase_Source_Table" ref="hbaseTable"/>
+								<ENTRY key="nameNode" env="nameNode"/>
+								<ENTRY key="jobTracker" env="jobTracker"/>
+								<ENTRY key="Stats_indexConf" env="index.entity.links"/>
+							</MAP>
+						</PARAM>
+
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="success"/>
+					</ARCS>
+				</NODE>
+
+			</WORKFLOW>
+
+		</CONFIGURATION>
+
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/metadata2hbase.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/metadata2hbase.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/metadata2hbase.xml	(revision 43444)
@@ -0,0 +1,95 @@
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="5b05a65a-4eeb-4862-bc55-b35c7ec3baf0_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2016-06-20T07:52:08+00:00"/>
+	</HEADER>
+	<BODY>
+		<CONFIGURATION>
+			<PARAMETERS>
+				<PARAM name="reuseMdRecords" description="reuse metadata records?" required="true" type="boolean"/>
+				<PARAM name="mdFormat" description="metadata format name" required="true" type="string"/>
+				<PARAM name="mapping" description="xslt mapping function" required="true" type="string"/>
+				<PARAM name="hdfsPath" description="HDFS path" required="true" type="string"/>
+				<PARAM name="hbaseTable" description="HBase table name" required="true" type="string"/>
+				<PARAM name="cluster" description="Hadoop cluster name" required="true" type="string"/>
+			</PARAMETERS>
+			<WORKFLOW>
+
+				<NODE name="reuseHdfsRecords" type="ReuseHdfsRecords" isStart="true">
+					<DESCRIPTION>reuse mdstore records</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="reuseMdRecords" ref="reuseMdRecords"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC name="true" to="doneExport"/>
+						<ARC name="false" to="exportRecords"/>
+					</ARCS>
+				</NODE>
+				<NODE name="exportRecords" type="MDStoreBatchExporter">
+					<DESCRIPTION>Fetch mdstore records</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="format" ref="mdFormat"/>
+						<PARAM name="layout" value="store"/>
+						<PARAM name="interpretation" value="cleaned"/>
+						<PARAM name="outputEprParam" value="records_epr"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="storeHdfsRecords"/>
+					</ARCS>
+				</NODE>
+				<NODE name="storeHdfsRecords" type="StoreHdfsRecords">
+					<DESCRIPTION>Store records to HDFS</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="inputEprParam" value="records_epr"/>
+						<PARAM name="hdfsPath" ref="hdfsPath"/>
+						<PARAM name="cluster" ref="cluster"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="doneExport"/>
+					</ARCS>
+				</NODE>
+				<NODE name="doneExport">
+					<DESCRIPTION/>
+					<PARAMETERS/>
+					<ARCS>
+						<ARC to="loadMapping"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="loadMapping" type="LoadMappingProfile">
+					<DESCRIPTION>Load mapping profile</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="profileId" ref="mapping"/>
+						<PARAM name="loadedMappingParam" value="hbase.import.xslt"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="mapreduce"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="mapreduce" type="SubmitHadoopJob">
+					<DESCRIPTION>Run M/R import Job</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="cluster" ref="cluster"/>
+						<PARAM name="hadoopJob" value="mdStoreHdfsImportJob"/>
+						<PARAM name="simulation" value="false"/>
+						<PARAM name="jobParams">
+							<MAP>
+								<ENTRY key="mapred.input.dir" ref="hdfsPath"/>
+								<ENTRY key="hbase.mapred.outputtable" ref="hbaseTable"/>
+								<ENTRY key="hbase.import.xslt" env="hbase.import.xslt"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="success"/>
+					</ARCS>
+				</NODE>
+
+			</WORKFLOW>
+		</CONFIGURATION>
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/entityRegistry/wds_store_er_template.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/entityRegistry/wds_store_er_template.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/entityRegistry/wds_store_er_template.xml	(revision 43444)
@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="311a298b-89c4-40b2-9043-3323c294e9dc_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2001-12-31T12:00:00"/>
+	</HEADER>
+	<BODY>
+		<CONFIGURATION>
+			<PARAMETERS>
+				<PARAM name="dsId" description="Datasource Id" required="true" type="string"/>
+				<PARAM name="interface" description="Datasource Interface" required="true" type="string"/>
+				<PARAM name="collMdstoreId" description="Store for collected records" required="true" type="string"/>
+				<PARAM name="cleanMdstoreId" description="Store for cleaned records" required="true" type="string"/>
+				<PARAM name="ruleId" description="Transformation Rule Id" required="true" type="string"/>
+			</PARAMETERS>
+			<WORKFLOW>
+
+
+				<NODE name="FETCH_RECORDS" type="FetchMDStoreRecords" isStart="true">
+					<DESCRIPTION>Fetch records from MDStore</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="mdId" ref="cleanMdstoreId"/>
+						<PARAM name="mdFormat" value="DSMF"/>
+						<PARAM name="eprParam" value="rs_epr"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="UPDATE_DB"/>
+					</ARCS>
+				</NODE>
+
+
+				<NODE name="UPDATE_DB" type="UpdateOpenaireDb">
+					<DESCRIPTION>Update Db</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="eprParam" value="rs_epr"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="success"/>
+					</ARCS>
+				</NODE>
+			</WORKFLOW>
+		</CONFIGURATION>
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/entityRegistry/wds_collection_er_template.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/entityRegistry/wds_collection_er_template.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/entityRegistry/wds_collection_er_template.xml	(revision 43444)
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="1fe60269-7e8e-49b8-a65c-f71491717a9f_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2001-12-31T12:00:00"/>
+	</HEADER>
+	<BODY>
+		<CONFIGURATION>
+			<PARAMETERS>
+				<PARAM name="dsId" description="Datasource Id" required="true" type="string"/>
+				<PARAM name="interface" description="Datasource Interface" required="true" type="string"/>
+				<PARAM name="collMdstoreId" description="Store for collected records" required="true" type="string"/>
+			</PARAMETERS>
+			<WORKFLOW>
+				<NODE name="COLLECT_REFRESH" type="CollectRecords" isStart="true">
+					<DESCRIPTION>Start Harvesting</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="datasourceId" ref="dsId"/>
+						<PARAM name="datasourceInterface" ref="interface"/>
+						<PARAM name="eprParam" value="collected_epr"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="MD_BUILDER_REFRESH"/>
+					</ARCS>
+				</NODE>
+				<NODE name="MD_BUILDER_REFRESH" type="MdBuilder">
+					<DESCRIPTION>Prepare mdstore records</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="inputEprParam" value="collected_epr"/>
+						<PARAM name="outputEprParam" value="store_epr"/>
+						<PARAM name="datasourceId" ref="dsId"/>
+						<PARAM name="datasourceInterface" ref="interface"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="STORE_REFRESH"/>
+					</ARCS>
+				</NODE>
+				<NODE name="STORE_REFRESH" type="StoreMDStoreRecords">
+					<DESCRIPTION>Store mdstore records</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="mdId" ref="collMdstoreId"/>
+						<PARAM name="storingType" value="REFRESH"/>
+						<PARAM name="eprParam" value="store_epr"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="UPDATE_INFO"/>
+					</ARCS>
+				</NODE>
+				<NODE name="UPDATE_INFO" type="MDStoreToApiExtraField">
+					<DESCRIPTION>Update datasouce API extra fields</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="mdId" ref="collMdstoreId"/>
+						<PARAM name="datasourceId" ref="dsId"/>
+						<PARAM name="datasourceInterface" ref="interface"/>
+						<PARAM name="extraFieldForTotal" value="last_collection_total"/>
+						<PARAM name="extraFieldForDate" value="last_collection_date"/>
+						<PARAM name="extraFieldForMdId" value="last_collection_mdId"/>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="success"/>
+					</ARCS>
+				</NODE>
+
+			</WORKFLOW>
+		</CONFIGURATION>
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/wds/collector/plugins/CMR2Datasource.st
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/wds/collector/plugins/CMR2Datasource.st	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/wds/collector/plugins/CMR2Datasource.st	(revision 43444)
@@ -0,0 +1,6 @@
+<datasource>
+	<id>$datasource.id$</id>
+	<name>$datasource.name$</name>
+	<url>$datasource.url$</url>
+	<uuid>$datasource.uuid$</uuid>
+</datasource>
Index: modules/dnet-wds/trunk/pom.xml
===================================================================
--- modules/dnet-wds/trunk/pom.xml	(revision 43443)
+++ modules/dnet-wds/trunk/pom.xml	(revision 43444)
@@ -16,19 +16,188 @@
     <developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet50/modules/dnet-wds/trunk</developerConnection>
   </scm>
 
-  <dependencies>
+	<!--<dependencies>-->
+	<!--<dependency>-->
+	<!--<groupId>junit</groupId>-->
+	<!--<artifactId>junit</artifactId>-->
+	<!--<version>${junit.version}</version>-->
+	<!--<scope>test</scope>-->
+	<!--</dependency>-->
+
+	<!--<dependency>-->
+	<!--<groupId>eu.dnetlib</groupId>-->
+	<!--<artifactId>dnet-msro-service</artifactId>-->
+	<!--<version>5.0.0-SNAPSHOT</version>-->
+	<!--</dependency>-->
+	<!--<dependency>-->
+	<!--<groupId>eu.dnetlib</groupId>-->
+	<!--<artifactId>dnet-openaire-domain</artifactId>-->
+	<!--<version>1.0.0-SNAPSHOT</version>-->
+	<!--</dependency>-->
+	<!--<dependency>-->
+	<!--<groupId>eu.dnetlib</groupId>-->
+	<!--<artifactId>cnr-data-flow-monitoring-core</artifactId>-->
+	<!--<version>2.0.1-java8</version>-->
+	<!--<exclusions>-->
+	<!--<exclusion>-->
+	<!--<artifactId>wstx-asl</artifactId>-->
+	<!--<groupId>org.codehaus.woodstox</groupId>-->
+	<!--</exclusion>-->
+	<!--</exclusions>-->
+	<!--</dependency>-->
+
+
+	<!--<dependency>-->
+	<!--<groupId>org.apache.velocity</groupId>-->
+	<!--<artifactId>velocity</artifactId>-->
+	<!--<version>1.7</version>-->
+	<!--<exclusions>-->
+	<!--<exclusion>-->
+	<!--<artifactId>antlr</artifactId>-->
+	<!--<groupId>antlr</groupId>-->
+	<!--</exclusion>-->
+	<!--<exclusion>-->
+	<!--<artifactId>commons-lang</artifactId>-->
+	<!--<groupId>commons-lang</groupId>-->
+	<!--</exclusion>-->
+	<!--</exclusions>-->
+	<!--</dependency>-->
+	<!--<dependency>-->
+	<!--<groupId>org.apache.velocity</groupId>-->
+	<!--<artifactId>velocity-tools</artifactId>-->
+	<!--<version>2.0</version>-->
+	<!--<exclusions>-->
+	<!--<exclusion>-->
+	<!--<artifactId>antlr</artifactId>-->
+	<!--<groupId>antlr</groupId>-->
+	<!--</exclusion>-->
+	<!--<exclusion>-->
+	<!--<artifactId>commons-beanutils</artifactId>-->
+	<!--<groupId>commons-beanutils</groupId>-->
+	<!--</exclusion>-->
+	<!--</exclusions>-->
+	<!--</dependency>-->
+	<!--<dependency>-->
+	<!--<groupId>org.apache.solr</groupId>-->
+	<!--<artifactId>solr-solrj</artifactId>-->
+	<!--<version>${apache.solr.version}</version>-->
+	<!--<exclusions>-->
+	<!--<exclusion>-->
+	<!--<artifactId>wstx-asl</artifactId>-->
+	<!--<groupId>org.codehaus.woodstox</groupId>-->
+	<!--</exclusion>-->
+	<!--<exclusion>-->
+	<!--<groupId>org.slf4j</groupId>-->
+	<!--<artifactId>jcl-over-slf4j</artifactId>-->
+	<!--</exclusion>-->
+	<!--<exclusion>-->
+	<!--<groupId>org.slf4j</groupId>-->
+	<!--<artifactId>slf4j-api</artifactId>-->
+	<!--</exclusion>-->
+
+	<!--</exclusions>-->
+	<!--</dependency>-->
+
+	<!--<dependency>-->
+	<!--<groupId>eu.dnetlib</groupId>-->
+	<!--<artifactId>dnet-mapreduce-jobs</artifactId>-->
+	<!--<version>1.0.0-SNAPSHOT</version>-->
+	<!--<scope>runtime</scope>-->
+	<!--</dependency>-->
+
+	<!--</dependencies>-->
+
+
+	<dependencies>
     <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <version>${junit.version}</version>
-      <scope>test</scope>
+	    <groupId>eu.dnetlib</groupId>
+	    <artifactId>dnet-msro-service</artifactId>
+	    <version>5.0.0-SNAPSHOT</version>
     </dependency>
+		<dependency>
+			<groupId>eu.dnetlib</groupId>
+			<artifactId>dnet-openaire-domain</artifactId>
+			<version>1.0.0-SNAPSHOT</version>
+		</dependency>
+		<dependency>
+			<groupId>eu.dnetlib</groupId>
+			<artifactId>cnr-data-flow-monitoring-core</artifactId>
+			<version>[2.0.0,3.0.0)</version>
+			<exclusions>
+				<exclusion>
+					<artifactId>wstx-asl</artifactId>
+					<groupId>org.codehaus.woodstox</groupId>
+				</exclusion>
+			</exclusions>
+		</dependency>
 
+
     <dependency>
+	    <groupId>org.apache.velocity</groupId>
+	    <artifactId>velocity</artifactId>
+	    <version>1.7</version>
+	    <exclusions>
+		    <exclusion>
+			    <artifactId>antlr</artifactId>
+			    <groupId>antlr</groupId>
+		    </exclusion>
+		    <exclusion>
+			    <artifactId>commons-lang</artifactId>
+			    <groupId>commons-lang</groupId>
+		    </exclusion>
+	    </exclusions>
+    </dependency>
+		<dependency>
+			<groupId>org.apache.velocity</groupId>
+			<artifactId>velocity-tools</artifactId>
+			<version>2.0</version>
+			<exclusions>
+				<exclusion>
+					<artifactId>antlr</artifactId>
+					<groupId>antlr</groupId>
+				</exclusion>
+				<exclusion>
+					<artifactId>commons-beanutils</artifactId>
+					<groupId>commons-beanutils</groupId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.solr</groupId>
+			<artifactId>solr-solrj</artifactId>
+			<version>${apache.solr.version}</version>
+			<exclusions>
+				<exclusion>
+					<artifactId>wstx-asl</artifactId>
+					<groupId>org.codehaus.woodstox</groupId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.slf4j</groupId>
+					<artifactId>jcl-over-slf4j</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.slf4j</groupId>
+					<artifactId>slf4j-api</artifactId>
+				</exclusion>
+
+			</exclusions>
+		</dependency>
+
+		<dependency>
       <groupId>eu.dnetlib</groupId>
-      <artifactId>dnet-core-components</artifactId>
+			<artifactId>dnet-mapreduce-jobs</artifactId>
       <version>1.0.0-SNAPSHOT</version>
+			<scope>runtime</scope>
     </dependency>
 
+
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<version>${junit.version}</version>
+			<scope>test</scope>
+		</dependency>
   </dependencies>
+
+
 </project>
Index: modules/dnet-data-support-services/trunk/src/main/java/eu/dnetlib/enabling/datasources/DnetPostgresDasourceManagerCoreImpl.java
===================================================================
--- modules/dnet-data-support-services/trunk/src/main/java/eu/dnetlib/enabling/datasources/DnetPostgresDasourceManagerCoreImpl.java	(revision 43443)
+++ modules/dnet-data-support-services/trunk/src/main/java/eu/dnetlib/enabling/datasources/DnetPostgresDasourceManagerCoreImpl.java	(revision 43444)
@@ -109,9 +109,14 @@
 
 		params.put("active", DatasourceFunctions.asSqlValue(active));
 		params.put("ifaceId", DatasourceFunctions.asSqlValue(ifaceId));
-		params.put("dsId", DatasourceFunctions.asSqlValue(dsId));
+		String correctDSId = dsId;
 
-		return datasourceManagerClients.updateSQL(dsId, "updateActivationStatus.sql.st", params, false, true);
+		if (StringUtils.endsWith(dsId, "_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=")) {
+			correctDSId = datasourceManagerClients.findDatasourceId(dsId);
+		}
+		params.put("dsId", DatasourceFunctions.asSqlValue(correctDSId));
+
+		return datasourceManagerClients.updateSQL(correctDSId, "updateActivationStatus.sql.st", params, false, true);
 	}
 
 	@Override
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/db2hbase.xml
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/db2hbase.xml	(nonexistent)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/db2hbase.xml	(revision 43444)
@@ -0,0 +1,212 @@
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="3ffe4cd8-73f1-4566-85fc-26ac32e8af56_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>
+		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>
+		<RESOURCE_URI value=""/>
+		<DATE_OF_CREATION value="2006-05-04T18:13:51.0Z"/>
+	</HEADER>
+	<BODY>
+		<CONFIGURATION>
+
+			<PARAMETERS>
+				<PARAM name="db" description="relational db name" required="true" type="string"/>
+				<PARAM name="sqlPrepareQueryDefault" description="the default prepare SQL query" type="string" default="/eu/dnetlib/msro/workflows/hbase/prepareQueryDefault.sql"/>
+				<PARAM name="sqlPrepareQueryDatasources" description="the prepare SQL query for datasources" type="string" default="/eu/dnetlib/msro/workflows/hbase/prepareQueryDatasources.sql"/>
+
+				<PARAM name="hbaseTable" description="HBase table name" required="true" type="string"/>
+				<PARAM name="cluster" description="Hadoop cluster name" required="true" type="string"/>
+
+				<PARAM name="sqlQueryDS" description="the SQL query" required="true" type="string"/>
+				<PARAM name="sqlQueryDSforSize" description="WTF" default="/eu/dnetlib/msro/workflows/hbase/queryDatasourcesForSize.sql"/>
+				<PARAM name="mappingDS" description="xslt mapping function" required="true" type="string"/>
+
+				<PARAM name="sqlQueryDS_ORG" description="the SQL query" required="true" type="string"/>
+				<PARAM name="mappingDS_ORG" description="xslt mapping function" required="true" type="string"/>
+
+				<PARAM name="sqlQueryORG" description="the SQL query" required="true" type="string"/>
+				<PARAM name="mappingORG" description="xslt mapping function" required="true" type="string"/>
+
+				<PARAM name="sqlQueryPERSON" description="the SQL query" required="true" type="string"/>
+				<PARAM name="mappingPERSON" description="xslt mapping function" required="true" type="string"/>
+
+				<PARAM name="sqlQueryPRJ_ORG" description="the SQL query" required="true" type="string"/>
+				<PARAM name="mappingPRJ_ORG" description="xslt mapping function" required="true" type="string"/>
+
+				<PARAM name="sqlQueryPRJ_PERSON" description="the SQL query" required="true" type="string"/>
+				<PARAM name="mappingPRJ_PERSON" description="xslt mapping function" required="true" type="string"/>
+
+				<PARAM name="sqlQueryPRJ" description="the SQL query" required="true" type="string"/>
+				<PARAM name="mappingPRJ" description="xslt mapping function" required="true" type="string"/>
+
+				<PARAM name="sqlQueryCLAIM_RELS" description="the SQL query" required="true" type="string"/>
+				<PARAM name="mappingCLAIM_RELS" description="xslt mapping function" required="true" type="string"/>
+			</PARAMETERS>
+
+			<WORKFLOW>
+
+				<NODE name="ds" type="LaunchWorkflowTemplate" isStart="true">
+					<DESCRIPTION>Datasource to HBase</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="9bb04513-be1d-4d13-8e86-785cc0375635_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="sqlQuery" ref="sqlQueryDS"/>
+								<ENTRY key="sqlForSize" ref="sqlQueryDSforSize"/>
+								<ENTRY key="mapping" ref="mappingDS"/>
+								<ENTRY key="sqlPrepareQuery" ref="sqlPrepareQueryDatasources"/>
+								<ENTRY key="db" ref="db"/>
+								<ENTRY key="hbaseTable" ref="hbaseTable"/>
+								<ENTRY key="cluster" ref="cluster"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="ds_org"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="ds_org" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>DatasourceOrganization to HBase</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="9bb04513-be1d-4d13-8e86-785cc0375635_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="sqlQuery" ref="sqlQueryDS_ORG"/>
+								<ENTRY key="mapping" ref="mappingDS_ORG"/>
+								<ENTRY key="sqlPrepareQuery" ref="sqlPrepareQueryDefault"/>
+								<ENTRY key="db" ref="db"/>
+								<ENTRY key="hbaseTable" ref="hbaseTable"/>
+								<ENTRY key="cluster" ref="cluster"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="org"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="org" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>DatasourceOrganization to HBase</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="9bb04513-be1d-4d13-8e86-785cc0375635_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="sqlQuery" ref="sqlQueryORG"/>
+								<ENTRY key="mapping" ref="mappingORG"/>
+								<ENTRY key="sqlPrepareQuery" ref="sqlPrepareQueryDefault"/>
+								<ENTRY key="db" ref="db"/>
+								<ENTRY key="hbaseTable" ref="hbaseTable"/>
+								<ENTRY key="cluster" ref="cluster"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="person"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="person" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>DatasourceOrganization to HBase</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="9bb04513-be1d-4d13-8e86-785cc0375635_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="sqlQuery" ref="sqlQueryPERSON"/>
+								<ENTRY key="mapping" ref="mappingPERSON"/>
+								<ENTRY key="sqlPrepareQuery" ref="sqlPrepareQueryDefault"/>
+								<ENTRY key="db" ref="db"/>
+								<ENTRY key="hbaseTable" ref="hbaseTable"/>
+								<ENTRY key="cluster" ref="cluster"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="prjOrg"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="prjOrg" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>DatasourceOrganization to HBase</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="9bb04513-be1d-4d13-8e86-785cc0375635_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="sqlQuery" ref="sqlQueryPRJ_ORG"/>
+								<ENTRY key="mapping" ref="mappingPRJ_ORG"/>
+								<ENTRY key="sqlPrepareQuery" ref="sqlPrepareQueryDefault"/>
+								<ENTRY key="db" ref="db"/>
+								<ENTRY key="hbaseTable" ref="hbaseTable"/>
+								<ENTRY key="cluster" ref="cluster"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="prjPerson"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="prjPerson" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>DatasourceOrganization to HBase</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="9bb04513-be1d-4d13-8e86-785cc0375635_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="sqlQuery" ref="sqlQueryPRJ_PERSON"/>
+								<ENTRY key="mapping" ref="mappingPRJ_PERSON"/>
+								<ENTRY key="sqlPrepareQuery" ref="sqlPrepareQueryDefault"/>
+								<ENTRY key="db" ref="db"/>
+								<ENTRY key="hbaseTable" ref="hbaseTable"/>
+								<ENTRY key="cluster" ref="cluster"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="project"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="project" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>DatasourceOrganization to HBase</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="9bb04513-be1d-4d13-8e86-785cc0375635_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="sqlQuery" ref="sqlQueryPRJ"/>
+								<ENTRY key="mapping" ref="mappingPRJ"/>
+								<ENTRY key="sqlPrepareQuery" ref="sqlPrepareQueryDefault"/>
+								<ENTRY key="db" ref="db"/>
+								<ENTRY key="hbaseTable" ref="hbaseTable"/>
+								<ENTRY key="cluster" ref="cluster"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="claimRels"/>
+					</ARCS>
+				</NODE>
+
+				<NODE name="claimRels" type="LaunchWorkflowTemplate">
+					<DESCRIPTION>DatasourceOrganization to HBase</DESCRIPTION>
+					<PARAMETERS>
+						<PARAM name="wfTemplateId" value="9bb04513-be1d-4d13-8e86-785cc0375635_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
+						<PARAM name="wfTemplateParams">
+							<MAP>
+								<ENTRY key="sqlQuery" ref="sqlQueryCLAIM_RELS"/>
+								<ENTRY key="mapping" ref="mappingCLAIM_RELS"/>
+								<ENTRY key="sqlPrepareQuery" ref="sqlPrepareQueryDefault"/>
+								<ENTRY key="db" ref="db"/>
+								<ENTRY key="hbaseTable" ref="hbaseTable"/>
+								<ENTRY key="cluster" ref="cluster"/>
+							</MAP>
+						</PARAM>
+					</PARAMETERS>
+					<ARCS>
+						<ARC to="success"/>
+					</ARCS>
+				</NODE>
+
+			</WORKFLOW>
+		</CONFIGURATION>
+	</BODY>
+</RESOURCE_PROFILE>
Index: modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/wds/workflows/repo-hi/wds_er_aggregation_wf.xml.st
===================================================================
--- modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/wds/workflows/repo-hi/wds_er_aggregation_wf.xml.st	(revision 43443)
+++ modules/dnet-wds/trunk/src/main/resources/eu/dnetlib/wds/workflows/repo-hi/wds_er_aggregation_wf.xml.st	(revision 43444)
@@ -19,13 +19,14 @@
             <PARAMETERS>
             	<PARAM name="collMdstoreId"         description="Store for collected records"               required="true" managedBy="system" category="MDSTORE_ID"/>
             	<PARAM name="cleanMdstoreId"        description="Store for cleaned records"                 required="true" managedBy="system" category="MDSTORE_ID"/>
-            	<PARAM name="cleanRuleId"           description="Cleaning Rule Identifier"                  required="true" managedBy="user"   category="TRANSFORMATION_RULE_ID" type="string" function="listProfiles('TransformationRuleDSResourceType', '//TITLE')"/>
+                <PARAM name="cleanRuleId"           description="Cleaning Rule Identifier"                  required="true" managedBy="user"   category="TRANSFORMATION_RULE_ID" type="string" function="listProfiles('CleanerDSResourceType', '//CLEANER_NAME')"/>
+            	<PARAM name="transformRuleId"       description="Transformation Rule Identifier"            required="true" managedBy="user"   category="TRANSFORMATION_RULE_ID" type="string" function="listProfiles('TransformationRuleDSResourceType', '//TITLE')"/>
             </PARAMETERS>
             <WORKFLOW>
                 <NODE isStart="true" name="collection" type="LaunchWorkflowTemplate">
                     <DESCRIPTION>Collect records</DESCRIPTION>
                     <PARAMETERS>
-                    	<PARAM name="wfTemplateId" value="<---------MISSING----------->" />
+                    	<PARAM name="wfTemplateId" value="1fe60269-7e8e-49b8-a65c-f71491717a9f_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ==" />
                     	<PARAM name="wfTemplateParams">
                     		<MAP>
                     			<ENTRY key="dsId"          value="$dsId$" />
@@ -35,20 +36,21 @@
                     	</PARAM>
                     </PARAMETERS>
                     <ARCS>
-                        <ARC to="clean"/>
+                        <ARC to="transform"/>
                     </ARCS>
                 </NODE>
                 <NODE name="transform" type="LaunchWorkflowTemplate">
                     <DESCRIPTION>Transform records</DESCRIPTION>
                     <PARAMETERS>
-                    	<PARAM name="wfTemplateId" value="<---------MISSING----------->" />
+                    	<PARAM name="wfTemplateId" value="276fa777-49eb-43f5-bf6f-c77751614f80_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ==" />
                     	<PARAM name="wfTemplateParams">
                     		<MAP>
-                    			<ENTRY key="dsId"           value="$dsId$" />
- 								<ENTRY key="interface"      value="$interface$" />
- 								<ENTRY key="collMdstoreId"  ref="collMdstoreId" />
- 								<ENTRY key="cleanMdstoreId" ref="cleanMdstoreId" />
- 								<ENTRY key="ruleId"         ref="cleanRuleId" />
+                    			<ENTRY key="dsId"               value="$dsId$" />
+ 								<ENTRY key="interface"          value="$interface$" />
+ 								<ENTRY key="collMdstoreId"      ref="collMdstoreId" />
+ 								<ENTRY key="cleanMdstoreId"     ref="cleanMdstoreId" />
+ 								<ENTRY key="transformRuleId"    ref="transformRuleId" />
+ 								<ENTRY key="cleanRuleId"        ref="cleanRuleId" />
  							</MAP>
  						</PARAM>
                     </PARAMETERS>
@@ -57,31 +59,10 @@
                     </ARCS>
                 </NODE>
 
-                <NODE name="transform" type="LaunchWorkflowTemplate">
-                    <DESCRIPTION>Transform records</DESCRIPTION>
-                    <PARAMETERS>
-                        <PARAM name="wfTemplateId" value="<---------MISSING----------->" />
-                        <PARAM name="wfTemplateParams">
-                            <MAP>
-                                <ENTRY key="dsId"           value="$dsId$" />
-                                <ENTRY key="interface"      value="$interface$" />
-                                <ENTRY key="collMdstoreId"  ref="collMdstoreId" />
-                                <ENTRY key="cleanMdstoreId" ref="cleanMdstoreId" />
-                                <ENTRY key="ruleId"         ref="cleanRuleId" />
-                            </MAP>
-                        </PARAM>
-                    </PARAMETERS>
-                    <ARCS>
-                        <ARC to="storeDatasource"/>
-                    </ARCS>
-                </NODE>
-
-
-
                 <NODE name="storeDatasource" type="LaunchWorkflowTemplate">
                     <DESCRIPTION>Transform records</DESCRIPTION>
                     <PARAMETERS>
-                        <PARAM name="wfTemplateId" value="<---------MISSING----------->" />
+                        <PARAM name="wfTemplateId" value="311a298b-89c4-40b2-9043-3323c294e9dc_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ==" />
                         <PARAM name="wfTemplateParams">
                             <MAP>
                                 <ENTRY key="dsId"           value="$dsId$" />
@@ -93,16 +74,17 @@
                         <ARC to="success"/>
                     </ARCS>
                 </NODE>
+                </WORKFLOW>
 
 	            <DESTROY_WORKFLOW_TEMPLATE id="1bd0b467-e6c1-40a5-a8f4-c344e0305ef6_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ==">
 	                <PARAMETERS>
 		                <PARAM name="dsId"                  value="$dsId$" />
 						<PARAM name="interface"             value="$interface$" />
-						<PARAM name="collMdstoreId"         ref="collMdstoreId" />
 						<PARAM name="cleanMdstoreId"        ref="cleanMdstoreId" />
 					</PARAMETERS>
 	            </DESTROY_WORKFLOW_TEMPLATE>
 
+
         </CONFIGURATION>
 
         <NOTIFICATIONS/>
Index: modules/dnet-openaire/trunk/pom.xml
===================================================================
--- modules/dnet-openaire/trunk/pom.xml	(revision 43443)
+++ modules/dnet-openaire/trunk/pom.xml	(revision 43444)
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
 	<parent>
 		<groupId>eu.dnetlib</groupId>
 		<artifactId>dnet-parent</artifactId>
@@ -104,4 +104,11 @@
 			<scope>test</scope>
 		</dependency>
 	</dependencies>
+	<repositories>
+		<repository>
+			<id>dfm-releases</id>
+			<url>http://maven.research-infrastructures.eu/nexus/content/repositories/dfm-releases/</url>
+		</repository>
+	</repositories>
+
 </project>
Index: modules/dnet-msro-service/trunk/src/main/resources/eu/dnetlib/msro/service/applicationContext-msro-nodes.xml
===================================================================
--- modules/dnet-msro-service/trunk/src/main/resources/eu/dnetlib/msro/service/applicationContext-msro-nodes.xml	(revision 43443)
+++ modules/dnet-msro-service/trunk/src/main/resources/eu/dnetlib/msro/service/applicationContext-msro-nodes.xml	(revision 43444)
@@ -180,6 +180,12 @@
 		class="eu.dnetlib.msro.workflows.nodes.misc.SetEnvParameterJobNode"
 		scope="prototype"/>
 
+
+	<bean id="wfNodeFindVocabularyEntries"
+	      class="eu.dnetlib.msro.workflows.nodes.misc.FindVocabularyEntriesJobNode"
+	      scope="prototype"
+	      p:xquery="classpath:/eu/dnetlib/msro/workflows/xqueries/vocabulary_entries.xquery"/>
+
 	<!-- Test -->
 	<bean id="wfNodeSleep"
 		class="eu.dnetlib.msro.workflows.nodes.misc.SleepJobNode"
