D-Net

package eu.dnetlib.msro.workflows.hadoop.hbase;

import java.util.Set;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.openaire.hadoop.utils.HBaseTableUtils;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

public class DefineHBaseOpenaireSchemaJobNode extends AbstractHBaseAdminJobNode {

	private static final Log log = LogFactory.getLog(DefineHBaseOpenaireSchemaJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	private String schema;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final String schemaOverride = StringUtils.isNotBlank(getSchema()) ? getSchema() : asCSV(HBaseTableUtils.listAllColumns());

		log.info("table definition: " + schemaOverride);

		token.getEnv().setAttribute(getTableColumnsParamName(), schemaOverride);

		return Arc.DEFAULT_ARC;

	}

	public String getSchema() {

		return schema;

	}

	public void setSchema(final String schema) {

		this.schema = schema;

	}

}

package eu.dnetlib.msro.workflows.hadoop.hbase;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.data.hadoop.rmi.HadoopService;

/**

 * The Class DropHBaseTableJobNode.

 */

public class DropHBaseTableJobNode extends AbstractHBaseAdminJobNode {

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(DropHBaseTableJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.msro.workflows.nodes.SimpleJobNode#execute(com.googlecode.sarasvati.NodeToken)

	 */

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final String tableName = tableName(token);

		final String cluster = cluster(token);

		log.info("Dropping hbase table '" + tableName + "' on cluster: '" + cluster + "'");

		getServiceLocator().getService(HadoopService.class).dropHbaseTable(cluster, tableName);

		return Arc.DEFAULT_ARC;

	}

}

package eu.dnetlib.msro.workflows.hadoop;

import java.io.IOException;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.rmi.MSROException;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

public class PrepareMDStoreImportJobNode extends SimpleJobNode {

	private static final Log log = LogFactory.getLog(PrepareMDStoreImportJobNode.class);

	@Autowired

	private UniqueServiceLocator serviceLocator;

	private String hdfsPathParam;

	private String hdfsPath;

	private String mappingParam;

	private String mapping;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		token.getEnv().setAttribute(getHdfsPathParam(), getHdfsPath());

		token.getEnv().setAttribute(getMappingParam(), readXslt(getMapping()));

		return Arc.DEFAULT_ARC;

	}

	private String readXslt(final String profileId) throws IOException, MSROException, ISLookUpException {

		if (StringUtils.isBlank(profileId)) throw new MSROException("missing profile id");

		log.info("loading mapping from profile id: " + profileId);

		final String xquery =

				String.format("/RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value='%s']/BODY/CONFIGURATION/SCRIPT/CODE/*[local-name()='stylesheet']", profileId);

		return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xquery);

	}

	public String getHdfsPathParam() {

		return hdfsPathParam;

	}

	public void setHdfsPathParam(final String hdfsPathParam) {

		this.hdfsPathParam = hdfsPathParam;

	}

	public String getHdfsPath() {

		return hdfsPath;

	}

	public void setHdfsPath(final String hdfsPath) {

		this.hdfsPath = hdfsPath;

	}

	public String getMapping() {

		return mapping;

	}

	public void setMapping(final String mapping) {

		this.mapping = mapping;

	}

	public String getMappingParam() {

		return mappingParam;

	}

	public void setMappingParam(final String mappingParam) {

		this.mappingParam = mappingParam;

	}

}

package eu.dnetlib.msro.workflows.hadoop.hbase;

import java.io.IOException;

import java.util.Map;

import com.googlecode.sarasvati.Engine;

import com.googlecode.sarasvati.NodeToken;

import com.googlecode.sarasvati.env.Env;

import eu.dnetlib.data.hadoop.rmi.HadoopBlackboardActions;

import eu.dnetlib.data.hadoop.rmi.HadoopService;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.enabling.resultset.rmi.ResultSetException;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions;

import eu.dnetlib.msro.rmi.MSROException;

import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;

import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;

import eu.dnetlib.msro.workflows.nodes.blackboard.BlackboardWorkflowJobListener;

import eu.dnetlib.msro.workflows.resultset.ProcessCountingResultSetFactory;

import eu.dnetlib.msro.workflows.util.ProgressProvider;

import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider;

import eu.dnetlib.msro.workflows.util.WorkflowsConstants;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

/**

 * Created by claudio on 08/04/16.

 */

public abstract class AbstractHBaseJobNode extends BlackboardJobNode implements ProgressJobNode {

	private static final Log log = LogFactory.getLog(StoreHBaseRecordsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	private final String INPUT_HBASE_TABLE_PARAM = "hbaseTable";

	private final String INPUT_EPR_PARAM = "input_epr";

	private final String INPUT_CLUSTER_PARAM = "cluster";

	private final String XSLT_PARAM = "xslt";

	private final String OUTPUT_HBASE_TABLE_PARAM = "table";

	private final String OUTPUT_CLUSTER_PARAM = "cluster";

	private final String SIMULATION_PARAM = "simulation";

	@Autowired

	protected UniqueServiceLocator serviceLocator;

	protected String inputEprParam;

	protected String hbaseTableProperty;

	protected String cluster;

	protected String mapping;

	protected boolean simulation = false;

	protected ProgressProvider progressProvider;

	protected ProcessCountingResultSetFactory processCountingResultSetFactory;

	protected abstract HadoopBlackboardActions getAction();

	@Override

	protected String obtainServiceId(final NodeToken token) {

		return getServiceLocator().getServiceId(HadoopService.class);

	}

	@Override

	protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {

		log.info("Invoking blackboard method: " + getAction().toString());

		job.setAction(getAction().toString());

		job.getParameters().put(INPUT_EPR_PARAM, DnetXsltFunctions.encodeBase64(prepareEpr(token)));

		job.getParameters().put(XSLT_PARAM, DnetXsltFunctions.encodeBase64(readXslt(getMapping())));

		job.getParameters().put(OUTPUT_HBASE_TABLE_PARAM, tableName(token));

		job.getParameters().put(OUTPUT_CLUSTER_PARAM, cluster(token));

		job.getParameters().put(SIMULATION_PARAM, String.valueOf(isSimulation()));

	}

	@Override

	protected BlackboardWorkflowJobListener generateBlackboardListener(final Engine engine, final NodeToken token) {

		return new BlackboardWorkflowJobListener(engine, token) {

			@Override

			protected void populateEnv(final Env env, final Map<String, String> responseParams) {

				final String count = responseParams.get("count");

				log.info(String.format("%s %s objects to HBase table %s, cluster %s", getAction().toString(), count, tableName(token), cluster(token)));

				env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + getName() + ":count", count);

			}

		};

	}

	protected String tableName(final NodeToken token) {

		if (token.getEnv().hasAttribute(INPUT_HBASE_TABLE_PARAM)) {

			String table = token.getEnv().getAttribute(INPUT_HBASE_TABLE_PARAM);

			log.debug("found override value in wfEnv for 'hbaseTable' param: " + table);

			return table;

		}

		return getPropertyFetcher().getProperty(getHbaseTableProperty());

	}

	protected String cluster(final NodeToken token) {

		if (token.getEnv().hasAttribute(INPUT_CLUSTER_PARAM)) {

			String cluster = token.getEnv().getAttribute("cluster");

			log.debug("found override value in wfEnv for 'cluster' param: " + cluster);

			return cluster;

		}

		return getCluster();

	}

	private String prepareEpr(final NodeToken token) throws ResultSetException {

		final String epr = token.getEnv().getAttribute(inputEprParam);

		final ResultsetProgressProvider resultsetProgressProvider = processCountingResultSetFactory.createProgressProvider(token.getProcess(), epr);

		setProgressProvider(resultsetProgressProvider);

		return resultsetProgressProvider.getEpr().toString();

	}

	private String readXslt(final String profileId) throws IOException, MSROException, ISLookUpException {

		if (StringUtils.isBlank(profileId)) throw new MSROException("missing profile id");

		log.info("loading mapping from profile id: " + profileId);

		final String xquery =

				String.format("/RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value='%s']/BODY/CONFIGURATION/SCRIPT/CODE/*[local-name()='stylesheet']", profileId);

		return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xquery);

	}

	public String getInputEprParam() {

		return inputEprParam;

	}

	public void setInputEprParam(final String inputEprParam) {

		this.inputEprParam = inputEprParam;

	}

	public String getHbaseTableProperty() {

		return hbaseTableProperty;

	}

	public void setHbaseTableProperty(final String hbaseTableProperty) {

		this.hbaseTableProperty = hbaseTableProperty;

	}

	@Override

	public ProgressProvider getProgressProvider() {

		return progressProvider;

	}

	public void setProgressProvider(final ProgressProvider progressProvider) {

		this.progressProvider = progressProvider;

	}

	public ProcessCountingResultSetFactory getProcessCountingResultSetFactory() {

		return processCountingResultSetFactory;

	}

	@Required

	public void setProcessCountingResultSetFactory(final ProcessCountingResultSetFactory processCountingResultSetFactory) {

		this.processCountingResultSetFactory = processCountingResultSetFactory;

	}

	public String getMapping() {

		return mapping;

	}

	public void setMapping(final String mapping) {

		this.mapping = mapping;

	}

	public String getCluster() {

		return cluster;

	}

	public void setCluster(final String cluster) {

		this.cluster = cluster;

	}

	public boolean isSimulation() {

		return simulation;

	}

	public void setSimulation(final boolean simulation) {

		this.simulation = simulation;

	}

}

package eu.dnetlib.msro.workflows.hadoop.hbase;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.data.hadoop.rmi.HadoopService;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

public class ExistHBaseTableJobNode extends AbstractHBaseAdminJobNode {

	private static final Log log = LogFactory.getLog(ExistHBaseTableJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	private String existOutNode;

	private String dontExistOutNode;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final String tableName = tableName(token);

		final String cluster = cluster(token);

		log.info("checking table existance: '" + tableName + "' on cluster: '" + cluster + "'");

		final HadoopService hadoopService = getServiceLocator().getService(HadoopService.class);

		boolean exists = hadoopService.existHbaseTable(cluster, tableName);

		log.info("table '" + tableName + "' exists: " + exists);

		if (exists) {

			final String tableDesc = hadoopService.describeHBaseTableConfiguration(cluster, tableName);

			token.getEnv().setAttribute(getTableConfigurationParamName(), tableDesc);

		}

		return exists ? getExistOutNode() : getDontExistOutNode();

	}

	public String getExistOutNode() {

		return existOutNode;

	}

	public void setExistOutNode(final String existOutNode) {

		this.existOutNode = existOutNode;

	}

	public String getDontExistOutNode() {

		return dontExistOutNode;

	}

	public void setDontExistOutNode(final String dontExistOutNode) {

		this.dontExistOutNode = dontExistOutNode;

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-deduplication</artifactId>

	<packaging>jar</packaging>

	<version>2.0.1</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-deduplication/tags/dnet-deduplication-2.0.1</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-msro-service</artifactId>

			<version>[4.0.0,5.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-hadoop-service-rmi</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-actionmanager-api</artifactId>

			<version>[4.0.0,5.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-modular-ui</artifactId>

			<version>[3.0.0,4.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-index-client</artifactId>

			<version>[3.0.0,4.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-openaireplus-mapping-utils</artifactId>

			<version>[7.0.0,8.0.0)</version>

			<exclusions>

				<exclusion>

					<groupId>eu.dnetlib</groupId>

					<artifactId>dnet-hadoop-commons</artifactId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>javax.servlet</groupId>

			<artifactId>javax.servlet-api</artifactId>

			<version>${javax.servlet.version}</version>

			<scope>provided</scope>

		</dependency>

		<dependency>

			<groupId>com.fasterxml.jackson.core</groupId>

			<artifactId>jackson-databind</artifactId>

			<version>${jackson.version}</version>

		</dependency>

		<dependency>

			<groupId>com.google.guava</groupId>

			<artifactId>guava</artifactId>

			<version>${google.guava.version}</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

	</dependencies>

</project>

package eu.dnetlib.msro.workflows.hadoop.hbase;

import java.util.Map.Entry;

import java.util.Set;

import javax.annotation.Resource;

import com.google.common.base.Joiner;

import com.google.common.base.Splitter;

import com.google.common.collect.Sets;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.rmi.MSROException;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

public abstract class AbstractHBaseAdminJobNode extends SimpleJobNode {

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(AbstractHBaseAdminJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	private String tableColumnsParamName = "columns";

	private String tableConfigurationParamName = "tableConf";

	private String hbaseTableProperty;

	private String cluster;

	@Resource

	private UniqueServiceLocator serviceLocator;

	@Override

	protected void beforeStart(final NodeToken token) {

		for (Entry<String, String> e : parseJsonParameters(token).entrySet()) {

			token.getEnv().setAttribute(e.getKey(), e.getValue());

		}

	}

	protected String tableName(final NodeToken token) {

		if (token.getEnv().hasAttribute("hbaseTable")) {

			String table = token.getEnv().getAttribute("hbaseTable");

			log.debug("found override value in wfEnv for 'hbaseTable' param: " + table);

			return table;

		}

		return getPropertyFetcher().getProperty(getHbaseTableProperty());

	}

	protected String cluster(final NodeToken token) {

		if (token.getEnv().hasAttribute("cluster")) {

			String cluster = token.getEnv().getAttribute("cluster");

			log.debug("found override value in wfEnv for 'cluster' param: " + cluster);

			return cluster;

		}

		return getCluster();

	}

	protected Set<String> getColumns(final NodeToken token) throws MSROException {

		String envCols = token.getEnv().getAttribute(getTableColumnsParamName());

		if (StringUtils.isBlank(envCols)) { throw new MSROException("cannot find table description"); }

		log.debug("using columns from env: " + envCols);

		return Sets.newHashSet(Splitter.on(",").omitEmptyStrings().split(envCols));

	}

	protected String asCSV(final Iterable<String> columns) {

		return Joiner.on(",").skipNulls().join(columns);

	}

	public String getCluster() {

		return cluster;

	}

	public void setCluster(final String cluster) {

		this.cluster = cluster;

	}

	public String getHbaseTableProperty() {

		return hbaseTableProperty;

	}

	public void setHbaseTableProperty(final String hbaseTableProperty) {

		this.hbaseTableProperty = hbaseTableProperty;

	}

	public String getTableColumnsParamName() {

		return tableColumnsParamName;

	}

	public void setTableColumnsParamName(final String tableColumnsParamName) {

		this.tableColumnsParamName = tableColumnsParamName;

	}

	public UniqueServiceLocator getServiceLocator() {

		return serviceLocator;

	}

	public String getTableConfigurationParamName() {

		return tableConfigurationParamName;

	}

	public void setTableConfigurationParamName(final String tableConfigurationParamName) {

		this.tableConfigurationParamName = tableConfigurationParamName;

	}

}

package eu.dnetlib.msro.workflows.hadoop;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.data.hadoop.rmi.HadoopBlackboardActions;

import eu.dnetlib.data.hadoop.rmi.HadoopService;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;

public class CreateHdfsDirectoryJobNode extends BlackboardJobNode {

	private String cluster;

	private boolean force = false;

	@Override

	protected String obtainServiceId(final NodeToken token) {

		return getServiceLocator().getServiceId(HadoopService.class);

	}

	@Override

	protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {

		job.setAction(HadoopBlackboardActions.CREATE_HDFS_DIR.toString());

		job.getParameters().put("cluster", getCluster());

		job.getParameters().put("force", String.valueOf(isForce()));

		// The "path" parameter is set by the following call

		job.getParameters().putAll(parseJsonParameters(token));

	}

	public String getCluster() {

		return cluster;

	}

	public void setCluster(final String cluster) {

		this.cluster = cluster;

	}

	public boolean isForce() {

		return force;

	}

	public void setForce(boolean force) {

		this.force = force;

	}

}

package eu.dnetlib.msro.workflows.hadoop;

import javax.annotation.Resource;

import org.apache.commons.lang.StringUtils;

import org.springframework.beans.factory.annotation.Value;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.data.hadoop.rmi.HadoopService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.workflows.nodes.AsyncJobNode;

/**

 * The Class SetClusterAndTableJobNode.

 */

public class SetClusterAndTableJobNode extends AsyncJobNode {

	/** The cluster. */

	private String cluster;

	/** The table. */

	@Value("${hbase.mapred.datatable}")

	private String table;

	/** The table param. */

	private String tableParam;

	/** The service locator. */

	@Resource

	private UniqueServiceLocator serviceLocator;

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.msro.workflows.nodes.AsyncJobNode#execute(com.googlecode.sarasvati.NodeToken)

	 */

	@Override

	protected String execute(final NodeToken token) throws Exception {

		if (StringUtils.isBlank(getCluster())) throw new IllegalArgumentException("missing cluster name parameter");

		if (StringUtils.isBlank(getTable())) throw new IllegalArgumentException("missing table name parameter");

		if (!serviceLocator.getService(HadoopService.class).existHbaseTable(getCluster(), getTable()))

			throw new IllegalArgumentException(String.format("unexisting table %s on cluster %s", getTable(), getCluster()));

		token.getEnv().setAttribute("cluster", getCluster());

		token.getEnv().setAttribute(getTableParam(), getTable());

		return Arc.DEFAULT_ARC;

	}

	/**

	 * Gets the cluster.

	 *

	 * @return the cluster

	 */

	public String getCluster() {

		return cluster;

	}

	/**

	 * Sets the cluster.

	 *

	 * @param cluster

	 *            the new cluster

	 */

	public void setCluster(final String cluster) {

		this.cluster = cluster;

	}

	/**

	 * Gets the table param.

	 *

	 * @return the table param

	 */

	public String getTableParam() {

		return tableParam;

	}

	/**

	 * Sets the table param.

	 *

	 * @param tableParam

	 *            the new table param

	 */

	public void setTableParam(final String tableParam) {

		this.tableParam = tableParam;

	}

	public String getTable() {

		return table;

	}

	public void setTable(String table) {

		this.table = table;

	}

}

package eu.dnetlib.msro.workflows.dedup.conf;

import static org.junit.Assert.assertNotNull;

import static org.junit.Assert.assertTrue;

import java.io.IOException;

import java.nio.charset.Charset;

import java.util.Queue;

import org.apache.commons.io.IOUtils;

import org.junit.Before;

import org.junit.Test;

import org.junit.Ignore;

import com.google.common.collect.Lists;

import eu.dnetlib.pace.config.DedupConfig;

@Ignore

public class DedupConfigurationOrchestrationTest {

	public DedupConfigurationOrchestration dco;

	@Before

	public void setUp() throws IOException {

		final Entity e = new Entity("result", "50", "Publication");

		final String actionSetId = "001";

		final Queue<DedupConfig> configurations = Lists.newLinkedList();

		configurations.add(DedupConfig.loadDefault());

		dco = new DedupConfigurationOrchestration(e, actionSetId, configurations);

		assertNotNull(dco);

		assertNotNull(dco.getActionSetId());

		assertNotNull(dco.getEntity());

		assertNotNull(dco.getConfigurations());

	}

	@Test

	public void testSerialization() {

		final String json = dco.toString();

		final DedupConfigurationOrchestration anotherDco = DedupConfigurationOrchestration.fromJSON(json);

		assertNotNull(anotherDco);

		assertTrue(json.equals(anotherDco.toString()));

	}

	@Test

	public void testSerializationOrgs() throws IOException {

		final Entity e = new Entity("organization", "20", "Organization");

		final String actionSetId = "001";

		final Queue<DedupConfig> configurations = Lists.newLinkedList();

		configurations.add(DedupConfig.load(IOUtils.toString(getClass().getResourceAsStream("organisation.conf.json"), Charset.forName("UTF-8"))));

		dco = new DedupConfigurationOrchestration(e, actionSetId, configurations);

		System.out.println(dco.toString());

	}

}

package eu.dnetlib.msro.workflows.hadoop;

import java.util.Map;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import com.googlecode.sarasvati.Engine;

import com.googlecode.sarasvati.NodeToken;

import com.googlecode.sarasvati.env.Env;

import eu.dnetlib.data.hadoop.rmi.HadoopBlackboardActions;

import eu.dnetlib.data.hadoop.rmi.HadoopService;

import eu.dnetlib.enabling.resultset.rmi.ResultSetException;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions;

import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;

import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;

import eu.dnetlib.msro.workflows.nodes.blackboard.BlackboardWorkflowJobListener;

import eu.dnetlib.msro.workflows.resultset.ProcessCountingResultSetFactory;

import eu.dnetlib.msro.workflows.util.ProgressProvider;

import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider;

import eu.dnetlib.msro.workflows.util.WorkflowsConstants;

public class StoreHdfsRecordsJobNode extends BlackboardJobNode implements ProgressJobNode {

	private static final Log log = LogFactory.getLog(StoreHdfsRecordsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	private String inputEprParam;

	private String hdfsPathParam;

	private String cluster;

	private ProgressProvider progressProvider;

	private ProcessCountingResultSetFactory processCountingResultSetFactory;

	@Override

	protected String obtainServiceId(final NodeToken token) {

		return getServiceLocator().getServiceId(HadoopService.class);

	}

	@Override

	protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {

		log.info("Invoking blackboard method");

		job.setAction(HadoopBlackboardActions.IMPORT_EPR_HDFS.toString());

		job.getParameters().put("input_epr", DnetXsltFunctions.encodeBase64(prepareEpr(token)));

		job.getParameters().put("path", token.getEnv().getAttribute(getHdfsPathParam()));

		job.getParameters().put("cluster", getCluster());

	}

	@Override

	protected BlackboardWorkflowJobListener generateBlackboardListener(final Engine engine, final NodeToken token) {

		return new BlackboardWorkflowJobListener(engine, token) {

			@Override

			protected void populateEnv(final Env env, final Map<String, String> responseParams) {

				env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + getName() + ":count", responseParams.get("count"));

			}

		};

	}

	private String prepareEpr(final NodeToken token) throws ResultSetException {

		final String epr = token.getEnv().getAttribute(inputEprParam);

		final ResultsetProgressProvider resultsetProgressProvider = processCountingResultSetFactory.createProgressProvider(token.getProcess(), epr);

		setProgressProvider(resultsetProgressProvider);

		return resultsetProgressProvider.getEpr().toString();

	}

	public String getInputEprParam() {

		return inputEprParam;

	}

	public void setInputEprParam(final String inputEprParam) {

		this.inputEprParam = inputEprParam;

	}

	@Required

	public void setProcessCountingResultSetFactory(final ProcessCountingResultSetFactory processCountingResultSetFactory) {

		this.processCountingResultSetFactory = processCountingResultSetFactory;

	}

	@Override

	public ProgressProvider getProgressProvider() {

		return progressProvider;

	}

	public void setProgressProvider(final ProgressProvider progressProvider) {

		this.progressProvider = progressProvider;

	}

	public ProcessCountingResultSetFactory getProcessCountingResultSetFactory() {

		return processCountingResultSetFactory;

	}

	public String getCluster() {

		return cluster;

	}

	public void setCluster(final String cluster) {

		this.cluster = cluster;

	}

	public String getHdfsPathParam() {

		return hdfsPathParam;

	}

	public void setHdfsPathParam(final String hdfsPathParam) {

		this.hdfsPathParam = hdfsPathParam;

	}

}

package eu.dnetlib.msro.workflows.dedup;

import java.io.IOException;

import com.google.common.collect.Iterables;

import com.googlecode.protobuf.format.JsonFormat;

import eu.dnetlib.data.proto.OafProtos.Oaf;

import org.apache.commons.codec.binary.Base64;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.junit.Test;

/**

 * Created by claudio on 05/04/16.

 */

public class SerializationTest {

	private static final Log log = LogFactory.getLog(SerializationTest.class);

	@Test

	public void test() throws IOException {

		final String data = Iterables.getFirst(IOUtils.readLines(getClass().getResourceAsStream("oaf_data.base64")), "");

		final byte[] oafBytes = Base64.decodeBase64(data);

		Oaf oaf = Oaf.parseFrom(oafBytes);

		JsonFormat jsonFormat = new JsonFormat();

		String asJson = jsonFormat.printToString(oaf);

		log.info(asJson);

	}

}

package eu.dnetlib.msro.workflows.hadoop.hbase;

import java.util.List;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.data.hadoop.rmi.HadoopService;

public class GetHBaseTableDescriptionJobNode extends AbstractHBaseAdminJobNode {

	private static final Log log = LogFactory.getLog(GetHBaseTableDescriptionJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final String tableName = tableName(token);

		final String cluster = cluster(token);

		log.info("getting table description: '" + tableName + "' on cluster: '" + cluster + "'");

		final List<String> columns = getServiceLocator().getService(HadoopService.class).describeHbaseTable(cluster, tableName);

		log.debug(String.format("table '%s': " + columns, tableName));

		token.getEnv().setAttribute(getTableColumnsParamName(), asCSV(columns));

		return Arc.DEFAULT_ARC;

	}

}

package eu.dnetlib.msro.workflows.dedup;

import java.util.List;

import org.junit.Before;

import org.junit.Test;

import com.google.common.collect.Lists;

import eu.dnetlib.data.proto.TypeProtos.Type;

import eu.dnetlib.msro.workflows.hadoop.utils.Similarity;

import eu.dnetlib.msro.workflows.hadoop.utils.SimilarityMeshBuilder;

public class SimilarityMeshBuilderTest {

	private List<String> list;

	@Before

	public void setUp() throws Exception {

		list = Lists.newArrayList();

		for (int i = 0; i < 10; i++) {

			list.add(i + "");

		}

	}

	@Test

	public void test() {

		final List<Similarity> combinations = SimilarityMeshBuilder.build(Type.result, list);

		System.out.println(combinations);

		System.out.println(combinations.size());

	}

}

package eu.dnetlib.data.dedup;

import com.google.common.collect.Lists;

import eu.dnetlib.data.proto.OafProtos.Oaf;

import com.google.common.base.Function;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.junit.Before;

import org.junit.Test;

import java.util.Map;

public class DedupInspectorFunctionsTest {

    private static final Log log = LogFactory.getLog(DedupInspectorFunctionsTest.class);

    private String orgFromIndex = "<record>CAESigMIFCK7ARK4ARIlCiNJbnRlcm5hdGlvbmFsIFRlbm5pcyBGZWRlcmF0aW9uIEx0ZCoHCgVm\r\nYWxzZTIHCgVmYWxzZToHCgVmYWxzZUIHCgVmYWxzZUoHCgVmYWxzZVIHCgVmYWxzZVoHCgVmYWxz\r\nZWIHCgVmYWxzZWoHCgVmYWxzZXIHCgVmYWxzZYIBNAoCR0ISDlVuaXRlZCBLaW5nZG9tGg5kbmV0\r\nOmNvdW50cmllcyIOZG5ldDpjb3VudHJpZXNCMnJjdWtfX19fX19fXzo6ODYxOTIxQzMtNjcyMy00\r\nOUMwLUIyQ0UtRDcyODJBOUMxRTk2SkkKMTEwfG9wZW5haXJlX19fXzo6YWIyZDMzMTA3NDFlYTgw\r\nZDNiODcyNmY2NTE1MDI4NTgSFFJlc2VhcmNoIENvdW5jaWxzIFVLWgoyMDE3LTExLTA0YjEyMHxy\r\nY3VrX19fX19fX186OjVjYjFmZTZhYjg1NDcwMTBiMjJkOTAyN2U3MjUyMzVkagoyMDE5LTA1LTE5\r\nIqsBCAEQARoDMC45IiRkZWR1cC1zaW1pbGFyaXR5LW9yZ2FuaXphdGlvbi1zaW1wbGUqegoic3lz\r\naW1wb3J0OmNyb3Nzd2FsazplbnRpdHlyZWdpc3RyeRIic3lzaW1wb3J0OmNyb3Nzd2FsazplbnRp\r\ndHlyZWdpc3RyeRoXZG5ldDpwcm92ZW5hbmNlX2FjdGlvbnMiF2RuZXQ6cHJvdmVuYW5jZV9hY3Rp\r\nb25z\r\n\n</record>";

    private DedupIndexDAO dao;

    @Before

    public void setUp() {

        dao = new DedupIndexDAO();

    }

    @Test

    public void test_1() {

        Oaf oaf = dao.getXml2OafFunction().apply(orgFromIndex);

        System.out.println(oaf);

        Map<String, String> map = dao.getOaf2FieldMapFunction("organization", Lists.newArrayList("legalname", "legalshortname", "country", "websiteurl", "provenance")).apply(oaf);

        System.out.println(map);

    }

}

package eu.dnetlib.data.dedup;

import java.util.List;

import java.util.Map.Entry;

import java.util.Set;

import javax.annotation.Resource;

import org.apache.commons.lang.exception.ExceptionUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Value;

import com.google.common.collect.Lists;

import eu.dnetlib.data.proto.TypeProtos.Type;

import eu.dnetlib.enabling.database.rmi.DatabaseService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.functionality.modular.ui.dedup.SimilarityGroup;

import eu.dnetlib.msro.workflows.hadoop.utils.Similarity;

import eu.dnetlib.msro.workflows.hadoop.utils.SimilarityMeshBuilder;

public class DedupDbDAO {

	private static final Log log = LogFactory.getLog(DedupDbDAO.class);

	@Value("${dnet.dedup.db.name}")

	private String dbName;

	@Resource

	private UniqueServiceLocator serviceLocator;

	public boolean commit(final SimilarityGroup group) throws Exception {

		final DatabaseService dbService = serviceLocator.getService(DatabaseService.class);

		log.info("adding similarities");

		updateGroupSql(dbService, group);

		log.info("adding dissimilarities");

		dissimilaritiesSql(dbService, group);

		return true;

	}

	private void updateGroupSql(final DatabaseService dbService, final SimilarityGroup group) throws Exception {

		log.info("cleanup similarities");

		for (final String id : group.getGroup()) {

			// sql.append(String.format("DELETE FROM entities WHERE objidentifier = '%s'; ", id));

			safeUpdateSql(dbService, dbName, String.format("DELETE FROM similarity_groups WHERE objidentifier = '%s'; ", id));

		}

		// cleanup empty groups

		log.info("cleanup empty groups");

		safeUpdateSql(dbService, dbName, "DELETE FROM groups WHERE id NOT IN (SELECT groupid FROM similarity_groups)");

		// look for dissimilarities to remove

		log.info("reverting dissimilarities");

		for (Similarity s : mesh(group)) {

			safeUpdateSql(dbService, dbName, String.format("DELETE FROM dissimilarities WHERE actionsetid = '%s' AND id1 = '%s' AND id2 = '%s'; ",

					group.getActionSet(), s.getPair().getKey(), s.getPair().getValue()));

		}

		log.info("adding new group, size: " + group.getGroup().size());

		final String type = group.getEntityType().getType();

		safeUpdateSql(dbService, dbName,

				String.format("INSERT INTO groups(id, entitytype, date, actionsetid) VALUES('%s', '%s', '%s', '%s'); ", group.getId(), type, group.getDate(),

						group.getActionSet()));

		for (final String id : group.getGroup()) {

			// add new entity if needed

			if (!dbService.contains(dbName, "entities", "id", id)) {

				safeUpdateSql(dbService, dbName, String.format("INSERT INTO entities(id, entitytype) VALUES('%s', '%s'); ", id, type));

			}

			// throw new Exception("id already defined in a similarity group.");

			safeUpdateSql(dbService, dbName, String.format("INSERT INTO similarity_groups(groupid, objidentifier) VALUES('%s', '%s'); ", group.getId(), id));

		}

	}

	private List<Similarity> mesh(final SimilarityGroup group) {

		return SimilarityMeshBuilder.build(Type.valueOf(group.getEntityType().getType()), Lists.newArrayList(group.getGroup()), false);

	}

	private void dissimilaritiesSql(final DatabaseService dbService, final SimilarityGroup group) throws Exception {

		final String type = group.getEntityType().getType();

		// add potential new entities

		for (final Entry<String, Set<String>> e : group.getDissimilar().entrySet()) {

			if (!dbService.contains(dbName, "entities", "id", e.getKey())) {

				safeUpdateSql(dbService, dbName, String.format("INSERT INTO entities(id, entitytype) VALUES('%s', '%s'); ", e.getKey(), type));

			}

			for (final String id : e.getValue()) {

				if (!dbService.contains(dbName, "entities", "id", id)) {

					safeUpdateSql(dbService, dbName, String.format("INSERT INTO entities(id, entitytype) VALUES('%s', '%s'); ", id, type));

				}

			}

		}

		log.info("store dissimilarities");

		for (final Entry<String, Set<String>> e : group.getDissimilar().entrySet()) {

			for (final String id : e.getValue()) {

				safeUpdateSql(dbService, dbName,

						String.format("INSERT INTO dissimilarities(id1, id2, actionsetid) VALUES('%s', '%s', '%s'); ", e.getKey(), id, group.getActionSet()));

			}

		}

	}

	private void safeUpdateSql(final DatabaseService dbService, final String dbName, final String sql) throws Exception {

		try {

			log.info(sql);

			dbService.updateSQL(dbName, sql);

		} catch (final Exception e) {

			log.error(e.getMessage());

			log.debug(ExceptionUtils.getFullStackTrace(e));

			throw e;

		}

	}

}

{

  "wf" : {

    "threshold" : "0.9",

    "dedupRun" : "001",

    "entityType" : "organization",

    "orderField" : "legalname",

    "queueMaxSize" : "2000",

    "groupMaxSize" : "50",

    "slidingWindowSize" : "200",

    "rootBuilder" : [ "organization", "projectOrganization_participation_isParticipant", "datasourceOrganization_provision_isProvidedBy" ],

    "includeChildren" : "true"

  },

  "pace" : {

    "clustering" : [

      { "name" : "sortedngrampairs", "fields" : [ "legalname" ], "params" : { "max" : 2, "ngramLen" : "3"} },

      { "name" : "suffixprefix", "fields" : [ "legalname" ], "params" : { "max" : 1, "len" : "3" } },

      { "name" : "urlclustering", "fields" : [ "websiteurl" ], "params" : { } }

    ],

    "strictConditions" : [

      { "name" : "exactMatch", "fields" : [ "gridid" ] }

    ],

    "conditions" : [

      { "name" : "exactMatch", "fields" : [ "country" ] },

      { "name" : "DomainExactMatch", "fields" : [ "websiteurl" ] }

    ],

    "model" : [

      { "name" : "country", "algo" : "Null", "type" : "String", "weight" : "0", "ignoreMissing" : "false", "path" : "organization/metadata/country/classid" },

      { "name" : "legalshortname", "algo" : "JaroWinklerNormalizedName", "type" : "String", "weight" : "0.1", "ignoreMissing" : "false", "path" : "organization/metadata/legalshortname/value" },

      { "name" : "legalname", "algo" : "JaroWinklerNormalizedName", "type" : "String", "weight" : "0.9", "ignoreMissing" : "false", "path" : "organization/metadata/legalname/value", "params" : {"windowSize" : 4, "threshold" : 0.5} },

      { "name" : "websiteurl", "algo" : "Null", "type" : "URL", "weight" : "0", "ignoreMissing" : "true", "path" : "organization/metadata/websiteurl/value", "params" : { "host" : 0.5, "path" : 0.5 } },

      { "name" : "gridid", "algo" : "Null", "type" : "String", "weight" : "0.0", "ignoreMissing" : "true", "path" : "pid[qualifier#classid = {grid}]/value" }

    ],

    "blacklists" : { }

  }

}

package eu.dnetlib.data.dedup;

import javax.annotation.Resource;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

public abstract class AbstractDedupDAO {

	@Resource

	protected UniqueServiceLocator serviceLocator;

}

package eu.dnetlib.msro.workflows.hadoop;

import javax.annotation.Resource;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.workflows.nodes.AsyncJobNode;

public class IndexDSUpdateJobNode extends AsyncJobNode {

	private static final Log log = LogFactory.getLog(IndexDSUpdateJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Resource

	private UniqueServiceLocator serviceLocator;

	@Override

	public String execute(final NodeToken token) throws Exception {

		final String dsId = token.getEnv().getAttribute("index_id");

		final String version = token.getEnv().getAttribute("index.feed.timestamp");

		log.info("updating indexDS: " + dsId + " version: " + version);

		updateIndexDS(dsId, version);

		return Arc.DEFAULT_ARC;

	}

	/**

	 * method updates the given indexDataStructureId INDEX_SIZE, INDEX_LAST_UPDATE

	 * 

	 * @param dsId

	 * @param version

	 * @return true if the update was performed successfully, false otherwise

	 * @throws ISRegistryException

	 */

	private boolean updateIndexDS(final String dsId, final String version) throws ISRegistryException {

		final String xquery = "for $x in collection('')/RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + dsId + "']"

				+ " return update value $x//INDEX_LAST_UPDATE with '" + version + "'";

		log.debug("\n\n updating indexDataStructure: " + xquery + "\n\n");

		return serviceLocator.getService(ISRegistryService.class).executeXUpdate(xquery);

	}

}

CAES6D0IMjrDDRLZAwpICghBcnRpY2xlcxI8CgdrZXl3b3JkEgdrZXl3b3JkGhNkbmV0OnJlc3VsdF9zdWJqZWN0IhNkbmV0OnJlc3VsdF9zdWJqZWN0Ev0BCrYBQW5hbHlzaXMgYW5kIFJlcG9ydCBvbiBPcmlnaW5hbCBEb2N1bWVudGFyeSBFdmlkZW5jZSBDb25jZXJuaW5nIHRoZSBVc2Ugb2YgT3BpdW0gaW4gSW5kaWE6IFtGdXJuaXNoZWQgdG8gdGhlIOKAnEJyaXRpc2ggTWVkaWNhbCBKb3VybmFs4oCdIGJ5IHVwd2FyZHMgb2YgMTAwIEluZGlhbiBNZWRpY2FsIE9mZmljZXJzLl0SQgoKbWFpbiB0aXRsZRIKbWFpbiB0aXRsZRoTZG5ldDpkYXRhQ2l0ZV90aXRsZSITZG5ldDpkYXRhQ2l0ZV90aXRsZSIMCgoxODk0LTAyLTE3QkoKC3B1YmxpY2F0aW9uEgtwdWJsaWNhdGlvbhoWZG5ldDpyZXN1bHRfdHlwb2xvZ2llcyIWZG5ldDpyZXN1bHRfdHlwb2xvZ2llc2IuCgNlbmcSB0VuZ2xpc2gaDmRuZXQ6bGFuZ3VhZ2VzIg5kbmV0Omxhbmd1YWdlc8oBAgoAMvgBGjkKBE9QRU4SC09wZW4gQWNjZXNzGhFkbmV0OmFjY2Vzc19tb2RlcyIRZG5ldDphY2Nlc3NfbW9kZXMiRQoEMDAwMRIHQXJ0aWNsZRoZZG5ldDpwdWJsaWNhdGlvbl9yZXNvdXJjZSIZZG5ldDpwdWJsaWNhdGlvbl9yZXNvdXJjZSpKCjExMHxvcGVuZG9hcl9fX186OjhiNmRkN2RiOWFmNDllNjczMDZmZWI1OWE4YmRjNTJjEhVFdXJvcGUgUHViTWVkIENlbnRyYWxKKGh0dHA6Ly9ldXJvcGVwbWMub3JnL2FydGljbGVzL1BNQzI0MDM4NTgy+AEaOQoET1BFThILT3BlbiBBY2Nlc3MaEWRuZXQ6YWNjZXNzX21vZGVzIhFkbmV0OmFjY2Vzc19tb2RlcyJFCgQwMDAxEgdBcnRpY2xlGhlkbmV0OnB1YmxpY2F0aW9uX3Jlc291cmNlIhlkbmV0OnB1YmxpY2F0aW9uX3Jlc291cmNlKkoKMTEwfG9wZW5kb2FyX19fXzo6OGI2ZGQ3ZGI5YWY0OWU2NzMwNmZlYjU5YThiZGM1MmMSFUV1cm9wZSBQdWJNZWQgQ2VudHJhbEooaHR0cDovL2V1cm9wZXBtYy5vcmcvYXJ0aWNsZXMvUE1DMjQwMzU3NzL4ARo5CgRPUEVOEgtPcGVuIEFjY2VzcxoRZG5ldDphY2Nlc3NfbW9kZXMiEWRuZXQ6YWNjZXNzX21vZGVzIkUKBDAwMDESB0FydGljbGUaGWRuZXQ6cHVibGljYXRpb25fcmVzb3VyY2UiGWRuZXQ6cHVibGljYXRpb25fcmVzb3VyY2UqSgoxMTB8b3BlbmRvYXJfX19fOjo4YjZkZDdkYjlhZjQ5ZTY3MzA2ZmViNTlhOGJkYzUyYxIVRXVyb3BlIFB1Yk1lZCBDZW50cmFsSihodHRwOi8vZXVyb3BlcG1jLm9yZy9hcnRpY2xlcy9QTUMyNDAzMzQyMvgBGjkKBE9QRU4SC09wZW4gQWNjZXNzGhFkbmV0OmFjY2Vzc19tb2RlcyIRZG5ldDphY2Nlc3NfbW9kZXMiRQoEMDAwMRIHQXJ0aWNsZRoZZG5ldDpwdWJsaWNhdGlvbl9yZXNvdXJjZSIZZG5ldDpwdWJsaWNhdGlvbl9yZXNvdXJjZSpKCjExMHxvcGVuZG9hcl9fX186OjhiNmRkN2RiOWFmNDllNjczMDZmZWI1OWE4YmRjNTJjEhVFdXJvcGUgUHViTWVkIENlbnRyYWxKKGh0dHA6Ly9ldXJvcGVwbWMub3JnL2FydGljbGVzL1BNQzI0MDM3OTMy+AEaOQoET1BFThILT3BlbiBBY2Nlc3MaEWRuZXQ6YWNjZXNzX21vZGVzIhFkbmV0OmFjY2Vzc19tb2RlcyJFCgQwMDAxEgdBcnRpY2xlGhlkbmV0OnB1YmxpY2F0aW9uX3Jlc291cmNlIhlkbmV0OnB1YmxpY2F0aW9uX3Jlc291cmNlKkoKMTEwfG9wZW5kb2FyX19fXzo6OGI2ZGQ3ZGI5YWY0OWU2NzMwNmZlYjU5YThiZGM1MmMSFUV1cm9wZSBQdWJNZWQgQ2VudHJhbEooaHR0cDovL2V1cm9wZXBtYy5vcmcvYXJ0aWNsZXMvUE1DMjQwMzQ1MEIYb2FpOmV1cm9wZXBtYy5vcmc6OTkzNzkyQhhvYWk6ZXVyb3BlcG1jLm9yZzo5OTM4MzdCGG9haTpldXJvcGVwbWMub3JnOjk5MzYyOUIYb2FpOmV1cm9wZXBtYy5vcmc6OTkzNDYxQhhvYWk6ZXVyb3BlcG1jLm9yZzo5OTM1MzZKSgoxMTB8b3BlbmRvYXJfX19fOjo4YjZkZDdkYjlhZjQ5ZTY3MzA2ZmViNTlhOGJkYzUyYxIVRXVyb3BlIFB1Yk1lZCBDZW50cmFsUkYKGG9haTpldXJvcGVwbWMub3JnOjk5Mzc5MhIqCgNvYWkSA29haRoOZG5ldDpwaWRfdHlwZXMiDmRuZXQ6cGlkX3R5cGVzUjgKClBNQzI0MDMzNDISKgoDcG1jEgNwbWMaDmRuZXQ6cGlkX3R5cGVzIg5kbmV0OnBpZF90eXBlc1JGChhvYWk6ZXVyb3BlcG1jLm9yZzo5OTM4MzcSKgoDb2FpEgNvYWkaDmRuZXQ6cGlkX3R5cGVzIg5kbmV0OnBpZF90eXBlc1I4CgpQTUMyNDAzNzkzEioKA3BtYxIDcG1jGg5kbmV0OnBpZF90eXBlcyIOZG5ldDpwaWRfdHlwZXNSRgoYb2FpOmV1cm9wZXBtYy5vcmc6OTkzNjI5EioKA29haRIDb2FpGg5kbmV0OnBpZF90eXBlcyIOZG5ldDpwaWRfdHlwZXNSOAoKUE1DMjQwMzQ1MBIqCgNwbWMSA3BtYxoOZG5ldDpwaWRfdHlwZXMiDmRuZXQ6cGlkX3R5cGVzUkYKGG9haTpldXJvcGVwbWMub3JnOjk5MzQ2MRIqCgNvYWkSA29haRoOZG5ldDpwaWRfdHlwZXMiDmRuZXQ6cGlkX3R5cGVzUjgKClBNQzI0MDM1NzcSKgoDcG1jEgNwbWMaDmRuZXQ6cGlkX3R5cGVzIg5kbmV0OnBpZF90eXBlc1I4CgpQTUMyNDAzODU4EioKA3BtYxIDcG1jGg5kbmV0OnBpZF90eXBlcyIOZG5ldDpwaWRfdHlwZXNSRgoYb2FpOmV1cm9wZXBtYy5vcmc6OTkzNTM2EioKA29haRIDb2FpGg5kbmV0OnBpZF90eXBlcyIOZG5ldDpwaWRfdHlwZXNaGDIwMTUtMDItMDZUMTM6MjU6MzMuODU0WmIxNTB8ZGVkdXBfd2ZfMDAxOjo0MTA0YTc3NDBhMTZhMjIzOWUzODcxYTEzYzZlNWQxY4IBlAgIMjrXBRLZAwpICghBcnRpY2xlcxI8CgdrZXl3b3JkEgdrZXl3b3JkGhNkbmV0OnJlc3VsdF9zdWJqZWN0IhNkbmV0OnJlc3VsdF9zdWJqZWN0Ev0BCrYBQW5hbHlzaXMgYW5kIFJlcG9ydCBvbiBPcmlnaW5hbCBEb2N1bWVudGFyeSBFdmlkZW5jZSBDb25jZXJuaW5nIHRoZSBVc2Ugb2YgT3BpdW0gaW4gSW5kaWE6IFtGdXJuaXNoZWQgdG8gdGhlIOKAnEJyaXRpc2ggTWVkaWNhbCBKb3VybmFs4oCdIGJ5IHVwd2FyZHMgb2YgMTAwIEluZGlhbiBNZWRpY2FsIE9mZmljZXJzLl0SQgoKbWFpbiB0aXRsZRIKbWFpbiB0aXRsZRoTZG5ldDpkYXRhQ2l0ZV90aXRsZSITZG5ldDpkYXRhQ2l0ZV90aXRsZSIMCgoxODk0LTAyLTAzQkoKC3B1YmxpY2F0aW9uEgtwdWJsaWNhdGlvbhoWZG5ldDpyZXN1bHRfdHlwb2xvZ2llcyIWZG5ldDpyZXN1bHRfdHlwb2xvZ2llc2IuCgNlbmcSB0VuZ2xpc2gaDmRuZXQ6bGFuZ3VhZ2VzIg5kbmV0Omxhbmd1YWdlc8oBAgoAMvgBGjkKBE9QRU4SC09wZW4gQWNjZXNzGhFkbmV0OmFjY2Vzc19tb2RlcyIRZG5ldDphY2Nlc3NfbW9kZXMiRQoEMDAwMRIHQXJ0aWNsZRoZZG5ldDpwdWJsaWNhdGlvbl9yZXNvdXJjZSIZZG5ldDpwdWJsaWNhdGlvbl9yZXNvdXJjZSpKCjExMHxvcGVuZG9hcl9fX186OjhiNmRkN2RiOWFmNDllNjczMDZmZWI1OWE4YmRjNTJjEhVFdXJvcGUgUHViTWVkIENlbnRyYWxKKGh0dHA6Ly9ldXJvcGVwbWMub3JnL2FydGljbGVzL1BNQzI0MDM4NThCGG9haTpldXJvcGVwbWMub3JnOjk5MzgzN0pKCjExMHxvcGVuZG9hcl9fX186OjhiNmRkN2RiOWFmNDllNjczMDZmZWI1OWE4YmRjNTJjEhVFdXJvcGUgUHViTWVkIENlbnRyYWxSOAoKUE1DMjQwMzg1OBIqCgNwbWMSA3BtYxoOZG5ldDpwaWRfdHlwZXMiDmRuZXQ6cGlkX3R5cGVzUkYKGG9haTpldXJvcGVwbWMub3JnOjk5MzgzNxIqCgNvYWkSA29haRoOZG5ldDpwaWRfdHlwZXMiDmRuZXQ6cGlkX3R5cGVzWhgyMDE1LTAyLTA2VDEzOjI0OjQzLjM1NVpiMTUwfG9kX19fX19fXzkwODo6OTQxMTFjYzhhNTYwMTgzOGQ5MjQ1NTFmZWViMTI4OTSKAQCCAZQICDI61wUS2QMKSAoIQXJ0aWNsZXMSPAoHa2V5d29yZBIHa2V5d29yZBoTZG5ldDpyZXN1bHRfc3ViamVjdCITZG5ldDpyZXN1bHRfc3ViamVjdBL9AQq2AUFuYWx5c2lzIGFuZCBSZXBvcnQgb24gT3JpZ2luYWwgRG9jdW1lbnRhcnkgRXZpZGVuY2UgQ29uY2VybmluZyB0aGUgVXNlIG9mIE9waXVtIGluIEluZGlhOiBbRnVybmlzaGVkIHRvIHRoZSDigJxCcml0aXNoIE1lZGljYWwgSm91cm5hbOKAnSBieSB1cHdhcmRzIG9mIDEwMCBJbmRpYW4gTWVkaWNhbCBPZmZpY2Vycy5dEkIKCm1haW4gdGl0bGUSCm1haW4gdGl0bGUaE2RuZXQ6ZGF0YUNpdGVfdGl0bGUiE2RuZXQ6ZGF0YUNpdGVfdGl0bGUiDAoKMTg5NC0wMi0xMEJKCgtwdWJsaWNhdGlvbhILcHVibGljYXRpb24aFmRuZXQ6cmVzdWx0X3R5cG9sb2dpZXMiFmRuZXQ6cmVzdWx0X3R5cG9sb2dpZXNiLgoDZW5nEgdFbmdsaXNoGg5kbmV0Omxhbmd1YWdlcyIOZG5ldDpsYW5ndWFnZXPKAQIKADL4ARo5CgRPUEVOEgtPcGVuIEFjY2VzcxoRZG5ldDphY2Nlc3NfbW9kZXMiEWRuZXQ6YWNjZXNzX21vZGVzIkUKBDAwMDESB0FydGljbGUaGWRuZXQ6cHVibGljYXRpb25fcmVzb3VyY2UiGWRuZXQ6cHVibGljYXRpb25fcmVzb3VyY2UqSgoxMTB8b3BlbmRvYXJfX19fOjo4YjZkZDdkYjlhZjQ5ZTY3MzA2ZmViNTlhOGJkYzUyYxIVRXVyb3BlIFB1Yk1lZCBDZW50cmFsSihodHRwOi8vZXVyb3BlcG1jLm9yZy9hcnRpY2xlcy9QTUMyNDAzNTc3QhhvYWk6ZXVyb3BlcG1jLm9yZzo5OTM2MjlKSgoxMTB8b3BlbmRvYXJfX19fOjo4YjZkZDdkYjlhZjQ5ZTY3MzA2ZmViNTlhOGJkYzUyYxIVRXVyb3BlIFB1Yk1lZCBDZW50cmFsUjgKClBNQzI0MDM1NzcSKgoDcG1jEgNwbWMaDmRuZXQ6cGlkX3R5cGVzIg5kbmV0OnBpZF90eXBlc1JGChhvYWk6ZXVyb3BlcG1jLm9yZzo5OTM2MjkSKgoDb2FpEgNvYWkaDmRuZXQ6cGlkX3R5cGVzIg5kbmV0OnBpZF90eXBlc1oYMjAxNS0wMi0wNlQxMzoyMDoyMi43ODhaYjE1MHxvZF9fX19fX185MDg6OmJhNGY4OGYwMjhiOWJkM2FlYzYxZjc2NGZkNTA4MGQzigEAggGUCAgyOtcFEtkDCkgKCEFydGljbGVzEjwKB2tleXdvcmQSB2tleXdvcmQaE2RuZXQ6cmVzdWx0X3N1YmplY3QiE2RuZXQ6cmVzdWx0X3N1YmplY3QS/QEKtgFBbmFseXNpcyBhbmQgUmVwb3J0IG9uIE9yaWdpbmFsIERvY3VtZW50YXJ5IEV2aWRlbmNlIENvbmNlcm5pbmcgdGhlIFVzZSBvZiBPcGl1bSBpbiBJbmRpYTogW0Z1cm5pc2hlZCB0byB0aGUg4oCcQnJpdGlzaCBNZWRpY2FsIEpvdXJuYWzigJ0gYnkgdXB3YXJkcyBvZiAxMDAgSW5kaWFuIE1lZGljYWwgT2ZmaWNlcnMuXRJCCgptYWluIHRpdGxlEgptYWluIHRpdGxlGhNkbmV0OmRhdGFDaXRlX3RpdGxlIhNkbmV0OmRhdGFDaXRlX3RpdGxlIgwKCjE4OTQtMDEtMDZCSgoLcHVibGljYXRpb24SC3B1YmxpY2F0aW9uGhZkbmV0OnJlc3VsdF90eXBvbG9naWVzIhZkbmV0OnJlc3VsdF90eXBvbG9naWVzYi4KA2VuZxIHRW5nbGlzaBoOZG5ldDpsYW5ndWFnZXMiDmRuZXQ6bGFuZ3VhZ2VzygECCgAy+AEaOQoET1BFThILT3BlbiBBY2Nlc3MaEWRuZXQ6YWNjZXNzX21vZGVzIhFkbmV0OmFjY2Vzc19tb2RlcyJFCgQwMDAxEgdBcnRpY2xlGhlkbmV0OnB1YmxpY2F0aW9uX3Jlc291cmNlIhlkbmV0OnB1YmxpY2F0aW9uX3Jlc291cmNlKkoKMTEwfG9wZW5kb2FyX19fXzo6OGI2ZGQ3ZGI5YWY0OWU2NzMwNmZlYjU5YThiZGM1MmMSFUV1cm9wZSBQdWJNZWQgQ2VudHJhbEooaHR0cDovL2V1cm9wZXBtYy5vcmcvYXJ0aWNsZXMvUE1DMjQwMzM0MkIYb2FpOmV1cm9wZXBtYy5vcmc6OTkzNDYxSkoKMTEwfG9wZW5kb2FyX19fXzo6OGI2ZGQ3ZGI5YWY0OWU2NzMwNmZlYjU5YThiZGM1MmMSFUV1cm9wZSBQdWJNZWQgQ2VudHJhbFI4CgpQTUMyNDAzMzQyEioKA3BtYxIDcG1jGg5kbmV0OnBpZF90eXBlcyIOZG5ldDpwaWRfdHlwZXNSRgoYb2FpOmV1cm9wZXBtYy5vcmc6OTkzNDYxEioKA29haRIDb2FpGg5kbmV0OnBpZF90eXBlcyIOZG5ldDpwaWRfdHlwZXNaGDIwMTUtMDItMDZUMTM6MjQ6NTQuMjE4WmIxNTB8b2RfX19fX19fOTA4Ojo0OGNiZmYxOTk1MDc0YzI0NmU2MGY1YzM5YmZiMzVjNooBAIIBlAgIMjrXBRLZAwpICghBcnRpY2xlcxI8CgdrZXl3b3JkEgdrZXl3b3JkGhNkbmV0OnJlc3VsdF9zdWJqZWN0IhNkbmV0OnJlc3VsdF9zdWJqZWN0Ev0BCrYBQW5hbHlzaXMgYW5kIFJlcG9ydCBvbiBPcmlnaW5hbCBEb2N1bWVudGFyeSBFdmlkZW5jZSBDb25jZXJuaW5nIHRoZSBVc2Ugb2YgT3BpdW0gaW4gSW5kaWE6IFtGdXJuaXNoZWQgdG8gdGhlIOKAnEJyaXRpc2ggTWVkaWNhbCBKb3VybmFs4oCdIGJ5IHVwd2FyZHMgb2YgMTAwIEluZGlhbiBNZWRpY2FsIE9mZmljZXJzLl0SQgoKbWFpbiB0aXRsZRIKbWFpbiB0aXRsZRoTZG5ldDpkYXRhQ2l0ZV90aXRsZSITZG5ldDpkYXRhQ2l0ZV90aXRsZSIMCgoxODk0LTAxLTEzQkoKC3B1YmxpY2F0aW9uEgtwdWJsaWNhdGlvbhoWZG5ldDpyZXN1bHRfdHlwb2xvZ2llcyIWZG5ldDpyZXN1bHRfdHlwb2xvZ2llc2IuCgNlbmcSB0VuZ2xpc2gaDmRuZXQ6bGFuZ3VhZ2VzIg5kbmV0Omxhbmd1YWdlc8oBAgoAMvgBGjkKBE9QRU4SC09wZW4gQWNjZXNzGhFkbmV0OmFjY2Vzc19tb2RlcyIRZG5ldDphY2Nlc3NfbW9kZXMiRQoEMDAwMRIHQXJ0aWNsZRoZZG5ldDpwdWJsaWNhdGlvbl9yZXNvdXJjZSIZZG5ldDpwdWJsaWNhdGlvbl9yZXNvdXJjZSpKCjExMHxvcGVuZG9hcl9fX186OjhiNmRkN2RiOWFmNDllNjczMDZmZWI1OWE4YmRjNTJjEhVFdXJvcGUgUHViTWVkIENlbnRyYWxKKGh0dHA6Ly9ldXJvcGVwbWMub3JnL2FydGljbGVzL1BNQzI0MDM3OTNCGG9haTpldXJvcGVwbWMub3JnOjk5Mzc5MkpKCjExMHxvcGVuZG9hcl9fX186OjhiNmRkN2RiOWFmNDllNjczMDZmZWI1OWE4YmRjNTJjEhVFdXJvcGUgUHViTWVkIENlbnRyYWxSOAoKUE1DMjQwMzc5MxIqCgNwbWMSA3BtYxoOZG5ldDpwaWRfdHlwZXMiDmRuZXQ6cGlkX3R5cGVzUkYKGG9haTpldXJvcGVwbWMub3JnOjk5Mzc5MhIqCgNvYWkSA29haRoOZG5ldDpwaWRfdHlwZXMiDmRuZXQ6cGlkX3R5cGVzWhgyMDE1LTAyLTA2VDEzOjI0OjQzLjEyMlpiMTUwfG9kX19fX19fXzkwODo6NDEwNGE3NzQwYTE2YTIyMzllMzg3MWExM2M2ZTVkMWOKAQCCAZQICDI61wUS2QMKSAoIQXJ0aWNsZXMSPAoHa2V5d29yZBIHa2V5d29yZBoTZG5ldDpyZXN1bHRfc3ViamVjdCITZG5ldDpyZXN1bHRfc3ViamVjdBL9AQq2AUFuYWx5c2lzIGFuZCBSZXBvcnQgb24gT3JpZ2luYWwgRG9jdW1lbnRhcnkgRXZpZGVuY2UgQ29uY2VybmluZyB0aGUgVXNlIG9mIE9waXVtIGluIEluZGlhOiBbRnVybmlzaGVkIHRvIHRoZSDigJxCcml0aXNoIE1lZGljYWwgSm91cm5hbOKAnSBieSB1cHdhcmRzIG9mIDEwMCBJbmRpYW4gTWVkaWNhbCBPZmZpY2Vycy5dEkIKCm1haW4gdGl0bGUSCm1haW4gdGl0bGUaE2RuZXQ6ZGF0YUNpdGVfdGl0bGUiE2RuZXQ6ZGF0YUNpdGVfdGl0bGUiDAoKMTg5NC0wMi0xN0JKCgtwdWJsaWNhdGlvbhILcHVibGljYXRpb24aFmRuZXQ6cmVzdWx0X3R5cG9sb2dpZXMiFmRuZXQ6cmVzdWx0X3R5cG9sb2dpZXNiLgoDZW5nEgdFbmdsaXNoGg5kbmV0Omxhbmd1YWdlcyIOZG5ldDpsYW5ndWFnZXPKAQIKADL4ARo5CgRPUEVOEgtPcGVuIEFjY2VzcxoRZG5ldDphY2Nlc3NfbW9kZXMiEWRuZXQ6YWNjZXNzX21vZGVzIkUKBDAwMDESB0FydGljbGUaGWRuZXQ6cHVibGljYXRpb25fcmVzb3VyY2UiGWRuZXQ6cHVibGljYXRpb25fcmVzb3VyY2UqSgoxMTB8b3BlbmRvYXJfX19fOjo4YjZkZDdkYjlhZjQ5ZTY3MzA2ZmViNTlhOGJkYzUyYxIVRXVyb3BlIFB1Yk1lZCBDZW50cmFsSihodHRwOi8vZXVyb3BlcG1jLm9yZy9hcnRpY2xlcy9QTUMyNDAzNDUwQhhvYWk6ZXVyb3BlcG1jLm9yZzo5OTM1MzZKSgoxMTB8b3BlbmRvYXJfX19fOjo4YjZkZDdkYjlhZjQ5ZTY3MzA2ZmViNTlhOGJkYzUyYxIVRXVyb3BlIFB1Yk1lZCBDZW50cmFsUjgKClBNQzI0MDM0NTASKgoDcG1jEgNwbWMaDmRuZXQ6cGlkX3R5cGVzIg5kbmV0OnBpZF90eXBlc1JGChhvYWk6ZXVyb3BlcG1jLm9yZzo5OTM1MzYSKgoDb2FpEgNvYWkaDmRuZXQ6cGlkX3R5cGVzIg5kbmV0OnBpZF90eXBlc1oYMjAxNS0wMi0wNlQxMzoyNTozMy44NTRaYjE1MHxvZF9fX19fX185MDg6OmMzYjJiMjA4OWE2YTJkMmUzNDZiZDg1YWQ5NmM3MzlligEAigEAIoEBCAEQABoDMC45IiJkZWR1cC1zaW1pbGFyaXR5LXJlc3VsdC1sZXZlbnN0ZWluKlIKD3N5c2ltcG9ydDpkZWR1cBIPc3lzaW1wb3J0OmRlZHVwGhZkbmV0OnByb3ZlbmFuY2VBY3Rpb25zIhZkbmV0OnByb3ZlbmFuY2VBY3Rpb25zKUT7BZlTAQAA

package eu.dnetlib.msro.workflows.hadoop.utils;

import com.google.gson.Gson;

import eu.dnetlib.data.proto.TypeProtos.Type;

import eu.dnetlib.miscutils.collections.Pair;

public class Similarity {

	private Pair<String, String> pair;

	private Type type;

	public Similarity(final Pair<String, String> pair, final Type type) {

		super();

		this.setPair(pair);

		this.setType(type);

	}

	public Pair<String, String> getPair() {

		return pair;

	}

	public void setPair(final Pair<String, String> pair) {

		this.pair = pair;

	}

	public Type getType() {

		return type;

	}

	public void setType(final Type type) {

		this.type = type;

	}

	@Override

	public String toString() {

		return new Gson().toJson(this, Similarity.class);

	}

}

package eu.dnetlib.msro.workflows.dedup.conf;

import java.io.IOException;

import java.util.Queue;

import com.google.gson.Gson;

import com.google.gson.GsonBuilder;

import eu.dnetlib.pace.config.DedupConfig;

import eu.dnetlib.pace.util.PaceException;

import org.codehaus.jackson.map.ObjectMapper;

/**

 * The Class DedupConfigurationOrchestration.

 */

public class DedupConfigurationOrchestration {

	/** The entity. */

	private Entity entity;

	/** The action set id. */

	private String actionSetId;

	/** The configurations. */

	private Queue<DedupConfig> configurations;

	public DedupConfigurationOrchestration() {}

	/**

	 * Instantiates a new dedup configuration orchestration.

	 *

	 * @param entity

	 *            the entity

	 * @param actionSetId

	 *            the action set id

	 * @param configurations

	 *            the configurations

	 */

	public DedupConfigurationOrchestration(final Entity entity, final String actionSetId, final Queue<DedupConfig> configurations) {

		super();

		this.setEntity(entity);

		this.setActionSetId(actionSetId);

		this.setConfigurations(configurations);

	}

	/**

	 * Gets the entity.

	 *

	 * @return the entity

	 */

	public Entity getEntity() {

		return entity;

	}

	/**

	 * Gets the action set id.

	 *

	 * @return the action set id

	 */

	public String getActionSetId() {

		return actionSetId;

	}

	/**

	 * Gets the configurations.

	 *

	 * @return the configurations

	 */

	public Queue<DedupConfig> getConfigurations() {

		return configurations;

	}

	public void setEntity(final Entity entity) {

		this.entity = entity;

	}

	public void setActionSetId(final String actionSetId) {

		this.actionSetId = actionSetId;

	}

	public void setConfigurations(final Queue<DedupConfig> configurations) {

		this.configurations = configurations;