D-Net

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:p="http://www.springframework.org/schema/p"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">

	<bean id="wfNodePrepareMDStoreImport"

		class="eu.dnetlib.msro.workflows.hadoop.PrepareMDStoreImportJobNode"

		scope="prototype" />

	<bean id="wfNodeReuseHdfsRecords"

		class="eu.dnetlib.msro.workflows.hadoop.ReuseHdfsRecordsJobNode"

		scope="prototype" />

	<bean id="wfNodeMDStoreBatchExporter"

		class="eu.dnetlib.msro.workflows.hadoop.MDStoreBatchExporterJobNode"

		scope="prototype" />		

	<bean id="wfNodeStoreHdfsRecords"

		class="eu.dnetlib.msro.workflows.hadoop.StoreHdfsRecordsJobNode"

		scope="prototype" p:processCountingResultSetFactory-ref="msroProcessCountingResultSetFactory" />		

	<bean id="wfNodeDeleteHdfsPathJob"

		class="eu.dnetlib.msro.workflows.hadoop.DeleteHdfsPathJobNode"

		scope="prototype" />		

	<bean id="wfNodeSubmitHadoopJob"

		class="eu.dnetlib.msro.workflows.hadoop.SubmitHadoopJobNode"

		scope="prototype" />

	<bean id="wfNodeSetClusterAndTable"

		class="eu.dnetlib.msro.workflows.hadoop.SetClusterAndTableJobNode"

		scope="prototype" />

	<bean id="wfNodeSetDedupConfiguration"

		class="eu.dnetlib.msro.workflows.dedup.DedupConfigurationSetterJobNode"

		scope="prototype" />

	<bean id="wfNodePrepareDedupIndexJob"

		class="eu.dnetlib.msro.workflows.dedup.PrepareDedupIndexJobNode"

		scope="prototype" p:hbaseTable="${hbase.mapred.datatable}" />

	<bean id="wfNodeFinalizeDedupIndexFeeding"

		class="eu.dnetlib.msro.workflows.dedup.FinalizeDedupIndexJobNode"

		scope="prototype" />				

	<!-- RESET -->		

	<bean id="wfNodeCheckHBaseTable"

		class="eu.dnetlib.msro.workflows.hadoop.hbase.ExistHBaseTableJobNode"

		scope="prototype" />

	<bean id="wfNodeDropHBaseTable"

		class="eu.dnetlib.msro.workflows.hadoop.hbase.DropHBaseTableJobNode"

		scope="prototype" />

	<bean id="wfNodeDefineHBaseOpenaireSchema"

		class="eu.dnetlib.msro.workflows.hadoop.hbase.DefineHBaseOpenaireSchemaJobNode"

		scope="prototype" />

	<bean id="wfNodeCreateHBaseTable"

		class="eu.dnetlib.msro.workflows.hadoop.hbase.CreateHBaseTableJobNode"

		scope="prototype" />

	<bean id="wfNodeFindIndex"

		class="eu.dnetlib.msro.workflows.hadoop.FindIndexJobNode"

		scope="prototype" />

	<bean id="wfNodeIndexDsUpdateJob"

		class="eu.dnetlib.msro.workflows.hadoop.IndexDSUpdateJobNode"

		scope="prototype" />

	<bean id="wfNodeQueryUserActionDbJob"

		class="eu.dnetlib.msro.workflows.dedup.QueryUserActionDbJobNode"

		scope="prototype" />

	<bean id="wfNodeBuildSimilarityMeshJob"

		class="eu.dnetlib.msro.workflows.dedup.BuildSimilarityMeshJobNode"

		p:similarity-ref="similarityTemplate" scope="prototype" />		

	<bean id="similarityTemplate"

		class="eu.dnetlib.springutils.stringtemplate.StringTemplateFactory"

		p:template="classpath:/eu/dnetlib/msro/workflows/dedup/similarityTemplate.st"

		scope="prototype" />	

	<bean id="wfNodeStoreHBase"

		class="eu.dnetlib.msro.workflows.hadoop.hbase.StoreHBaseRecordsJobNode"

		scope="prototype" p:processCountingResultSetFactory-ref="msroProcessCountingResultSetFactory" />		

	<bean id="wfNodeDeleteFromHBase"

		class="eu.dnetlib.msro.workflows.hadoop.hbase.DeleteHBaseRecordsJobNode"

		scope="prototype" p:processCountingResultSetFactory-ref="msroProcessCountingResultSetFactory" />		

	<bean id="wfNodePromoteActions"

		class="eu.dnetlib.msro.workflows.actions.PromoteActionsJobNode"

		scope="prototype" />

	<!-- DEDUP -->

	<bean id="dedupOrchestrationLoader"

		class="eu.dnetlib.msro.workflows.dedup.conf.DedupConfigurationOrchestrationLoader"

		scope="prototype" />

	<bean id="wfNodeCheckEntitySequence"

		class="eu.dnetlib.msro.workflows.dedup.DedupCheckEntitySequenceJobNode"

		scope="prototype" />

	<bean id="wfNodeFetchRelClasses"

		class="eu.dnetlib.msro.workflows.hadoop.FetchRelClassesJobNode"

		scope="prototype" />

	<bean id="wfNodeDedupSimilarityToActionsJobNode" class="eu.dnetlib.msro.workflows.dedup.DedupSimilarityToActionsJobNode"

		scope="prototype" />

	<bean id="wfNodeDedupGrouperJob" class="eu.dnetlib.msro.workflows.dedup.DedupGrouperJobNode"

		scope="prototype" />

	<bean id="wfNodeDuplicateScanJob" class="eu.dnetlib.msro.workflows.dedup.DedupDuplicateScanJobNode"

		scope="prototype"/>

	<bean id="wfNodeDedupConfigurationAwareJob"

		class="eu.dnetlib.msro.workflows.dedup.DedupConfigurationAwareJobNode"

		scope="prototype" />

	<!-- / DEDUP -->	

	<bean id="wfNodeUpdateActionSets" 

		class="eu.dnetlib.msro.workflows.actions.UpdateSetsJobNode"

		scope="prototype" />

	<bean id="wfNodePrepareActionSets" 

		class="eu.dnetlib.msro.workflows.actions.PrepareActionSetsJobNode"

		scope="prototype" />

	<bean id="wfNodePrepareConfiguredActionSet" 

		class="eu.dnetlib.msro.workflows.actions.PrepareConfiguredActionSetJobNode"

		scope="prototype" />

	<bean id="wfNodeCleanActionSetsProfile"

		class="eu.dnetlib.msro.workflows.actions.CleanActionSetsProfileJobNode"

		p:xupdate="${dnet.actionmanager.garbage.sets.xupdate}" scope="prototype" />

	<bean id="wfNodeGarbageSets"

		class="eu.dnetlib.msro.workflows.actions.GarbageSetsJobNode"

		scope="prototype" />		

</beans>

package eu.dnetlib.msro.workflows.dedup.conf;

import java.io.StringReader;

import java.util.List;

import java.util.Queue;

import javax.annotation.Resource;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.Element;

import org.dom4j.io.SAXReader;

import com.google.common.collect.Lists;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpDocumentNotFoundException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.pace.config.DedupConfig;

/**

 * The Class DedupConfigurationOrchestrationLoader.

 */

public class DedupConfigurationOrchestrationLoader {

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(DedupConfigurationOrchestrationLoader.class);

	/** The service locator. */

	@Resource

	private UniqueServiceLocator serviceLocator;

	/**

	 * Load the dedup orchestration profile from the IS.

	 *

	 * @param id

	 *            the id

	 * @return the dedup configuration orchestration

	 * @throws ISLookUpDocumentNotFoundException

	 *             the IS look up document not found exception

	 * @throws ISLookUpException

	 *             the IS look up exception

	 * @throws DocumentException

	 *             the document exception

	 */

	public DedupConfigurationOrchestration loadByActionSetId(final String id) throws Exception {

		final ISLookUpService isLookUpService = serviceLocator.getService(ISLookUpService.class);

		final String xquery = String.format("/RESOURCE_PROFILE[.//DEDUPLICATION/ACTION_SET/@id = '%s']", id);

		log.info("loading dedup orchestration: " + xquery);

		return parseOrchestrationProfile(isLookUpService, isLookUpService.getResourceProfileByQuery(xquery));

	}

	public List<DedupConfigurationOrchestration> loadByEntityName(final String entityName) throws Exception {

		final ISLookUpService isLookUpService = serviceLocator.getService(ISLookUpService.class);

		final String xquery = String.format("/RESOURCE_PROFILE[.//DEDUPLICATION/ENTITY/@name = '%s']", entityName);

		log.info("loading dedup orchestration: " + xquery);

		final List<DedupConfigurationOrchestration> res = Lists.newArrayList();

		for (final String profile : isLookUpService.quickSearchProfile(xquery)) {

			res.add(parseOrchestrationProfile(isLookUpService, profile));

		}

		return res;

	}

	private DedupConfigurationOrchestration parseOrchestrationProfile(final ISLookUpService isLookUpService, final String dedupOrchestation)

			throws DocumentException,

			ISLookUpException, ISLookUpDocumentNotFoundException {

		final Document doc = new SAXReader().read(new StringReader(dedupOrchestation));

		final Element e = (Element) doc.selectSingleNode("//DEDUPLICATION/ENTITY");

		final Entity entity = new Entity(e.attributeValue("name"), e.attributeValue("code"), e.attributeValue("label"));

		final String actionSetId = doc.valueOf("//DEDUPLICATION/ACTION_SET/@id");

		final Queue<DedupConfig> configurations = Lists.newLinkedList();

		for (final Object o : doc.selectNodes("//SCAN_SEQUENCE/SCAN")) {

			configurations.add(loadConfig(isLookUpService, actionSetId, o));

		}

		final DedupConfigurationOrchestration dco = new DedupConfigurationOrchestration(entity, actionSetId, configurations);

		log.debug("loaded dedup configuration orchestration: " + dco.toString());

		log.info("loaded dedup configuration orchestration, size: " + dco.getConfigurations().size());

		return dco;

	}

	private DedupConfig loadConfig(final ISLookUpService isLookUpService, final String actionSetId, final Object o) throws ISLookUpException,

			ISLookUpDocumentNotFoundException {

		final Element s = (Element) o;

		final String configProfileId = s.attributeValue("id");

		final String conf =

				isLookUpService.getResourceProfileByQuery(String.format(

						"for $x in /RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '%s'] return $x//DEDUPLICATION/text()",

						configProfileId));

		final DedupConfig dedupConfig = DedupConfig.load(conf);

		dedupConfig.getWf().setConfigurationId(actionSetId);

		return dedupConfig;

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:context="http://www.springframework.org/schema/context"

	xmlns:p="http://www.springframework.org/schema/p" xmlns:util="http://www.springframework.org/schema/util"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd

		http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd

		http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">

	<bean id="listDedupOrchestrationValues" lazy-init="true" 

		class="eu.dnetlib.functionality.modular.ui.workflows.values.ListDedupOrchestrationValues"

		p:name="dedupOrchestrations" />      

	<bean id="listDedupConfigValues" lazy-init="true" 

		class="eu.dnetlib.functionality.modular.ui.workflows.values.ListDedupConfigValues"

		p:name="dedupConfigs" />    

	<bean id="listActionManagerSetsValues" lazy-init="true" 

		class="eu.dnetlib.functionality.modular.ui.workflows.values.ListActionManagerSetsValues"

		p:name="actionSets" />

	<bean id="listHBaseTables" lazy-init="true"

		class="eu.dnetlib.functionality.modular.ui.workflows.values.ListHBaseTables"

		p:name="hbaseTables"/>

	<bean id="listHadoopClusters" lazy-init="true"

		class="eu.dnetlib.functionality.modular.ui.workflows.values.ListHadoopClusters"

		p:name="hadoopClusters" />	

	<bean id="listHBaseMappings" lazy-init="true"

		class="eu.dnetlib.functionality.modular.ui.workflows.values.ListHBaseMappingTitleValues"

		p:name="hbaseMappings" />

</beans>

package eu.dnetlib.msro.workflows.dedup;

import java.util.Queue;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import com.google.common.base.Function;

import com.google.common.base.Splitter;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.msro.rmi.MSROException;

import eu.dnetlib.msro.workflows.dedup.conf.DedupConfigurationOrchestration;

import eu.dnetlib.msro.workflows.dedup.conf.DedupConfigurationOrchestrationLoader;

import eu.dnetlib.msro.workflows.nodes.AsyncJobNode;

public class DedupCheckEntitySequenceJobNode extends AsyncJobNode {

	private static final Log log = LogFactory.getLog(DedupCheckEntitySequenceJobNode.class);

	@Autowired

	private DedupConfigurationOrchestrationLoader dedupOrchestrationLoader;

	private String dedupConfigSequenceParam;

	private String entitySequence;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		if (StringUtils.isBlank(getEntitySequence())) throw new MSROException("missing entity sequence, e.g. a csv: organization,person,result");

		if (token.getFullEnv().hasAttribute(DedupGrouperJobNode.DEDUP_GROUPER_LOOPER)) {

			log.info("reset env variable: " + DedupGrouperJobNode.DEDUP_GROUPER_LOOPER + " to zero");

			token.getFullEnv().setAttribute(DedupGrouperJobNode.DEDUP_GROUPER_LOOPER, 0);

		}

		if (!token.getEnv().hasAttribute("entitySequence")) {

			log.info("parsing config sequence: " + getEntitySequence());

			token.getEnv().setAttribute("entitySequence", getEntitySequence());

			final Iterable<String> sequence = Splitter.on(",").omitEmptyStrings().split(getEntitySequence());

			final Queue<DedupConfigurationOrchestration> q =

					Lists.newLinkedList(Iterables.transform(sequence, new Function<String, DedupConfigurationOrchestration>() {

						@Override

						public DedupConfigurationOrchestration apply(final String entityName) {

							try {

								final DedupConfigurationOrchestration dco = Iterables.getFirst(dedupOrchestrationLoader.loadByEntityName(entityName), null);

								if (dco == null) throw new RuntimeException("unable to find DedupOrchestration profile for entity type: " + entityName);

								return dco;

							} catch (final Throwable e) {

								throw new RuntimeException("", e);

							}

						}

					}));

			log.info("built sequence of dedup orchestration profiles, size: " + q.size());

			final DedupConfigurationOrchestration dco = q.remove();

			log.info("closing mesh for entity: " + dco.getEntity().getName());

			setDedupConfParams(token, dco);

			token.getEnv().setTransientAttribute("entitySequenceQueue", q);

			return Arc.DEFAULT_ARC;

		}

		@SuppressWarnings("unchecked")

		final Queue<DedupConfigurationOrchestration> q = (Queue<DedupConfigurationOrchestration>) token.getEnv().getTransientAttribute("entitySequenceQueue");

		if (!q.isEmpty()) {

			log.info("remaining dedup orchestration profiles: " + q.size());

			final DedupConfigurationOrchestration dco = q.remove();

			log.info("closing mesh for entity: " + dco.getEntity().getName());

			setDedupConfParams(token, dco);

			return Arc.DEFAULT_ARC;

		}

		log.info("completed closing mesh for entities: " + getEntitySequence());

		return "done";

	}

	private void setDedupConfParams(final NodeToken token, final DedupConfigurationOrchestration dco) {

		token.getEnv().setAttribute("entityType", dco.getEntity().getName());

		token.getEnv().setAttribute("entityTypeId", dco.getEntity().getCode());

		token.getEnv().setAttribute(getDedupConfigSequenceParam(), dco.toString());

	}

	public String getEntitySequence() {

		return entitySequence;

	}

	public void setEntitySequence(final String entitySequence) {

		this.entitySequence = entitySequence;

	}

	public String getDedupConfigSequenceParam() {

		return dedupConfigSequenceParam;

	}

	public void setDedupConfigSequenceParam(final String dedupConfigSequenceParam) {

		this.dedupConfigSequenceParam = dedupConfigSequenceParam;

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws"

	xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing"

	xmlns:p="http://www.springframework.org/schema/p" xmlns:http="http://cxf.apache.org/transports/http/configuration"

	xmlns:t="http://dnetlib.eu/springbeans/t" xmlns:template="http://dnetlib.eu/springbeans/template"

	xmlns:util="http://www.springframework.org/schema/util" xmlns:context="http://www.springframework.org/schema/context"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd

						http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd

						http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd

						http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd

						http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd

						http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd

						http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd

						http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">

	<bean name="/ui/dedupInspector.do"

		class="eu.dnetlib.functionality.modular.ui.dedup.DedupServiceEntryPointController"

		p:menu="Dedup Service Inspector" 

		p:title="Dedup Service Inspector"

		p:description="Dedup Service Inspector"

		p:order="5" 

		p:group="Tools">

		<property name="permissionLevels">

			<set>

				<value>IS_ADMIN</value>

			</set>

		</property>

	</bean>

</beans>

package eu.dnetlib.msro.workflows.dedup;

import org.apache.commons.collections.CollectionUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.Engine;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.dedup.conf.DedupConfigurationOrchestration;

import eu.dnetlib.msro.workflows.nodes.blackboard.BlackboardWorkflowJobListener;

public class DedupDuplicateScanJobNode extends DedupConfigurationAwareJobNode {

	private static final Log log = LogFactory.getLog(DedupDuplicateScanJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	private class DedupBlackboardWorkflowJobListener extends BlackboardWorkflowJobListener {

		public DedupBlackboardWorkflowJobListener(final Engine engine, final NodeToken token) {

			super(engine, token);

		}

		@Override

		protected void onDone(final BlackboardJob job) {

			final DedupConfigurationOrchestration confs = dedupConfigurations(getToken());

			confs.getConfigurations().poll();

			log.info("checking dedup configs queue, size: " + confs.getConfigurations().size());

			if (CollectionUtils.isEmpty(confs.getConfigurations())) {

				log.info("dedup similarity scan done");

				super.complete(job, "done");

			} else {

				log.debug("remaining confs: " + confs);

				getToken().getEnv().setAttribute(getDedupConfigSequenceParam(), confs.toString());

				super.complete(job, Arc.DEFAULT_ARC);

			}

		}

	}

	@Override

	protected BlackboardWorkflowJobListener generateBlackboardListener(final Engine engine, final NodeToken token) {

		return new DedupBlackboardWorkflowJobListener(engine, token);

	}

}

package eu.dnetlib.msro.workflows.dedup;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.data.proto.DedupSimilarityProtos.DedupSimilarity;

import eu.dnetlib.data.proto.RelTypeProtos.RelType;

import eu.dnetlib.data.proto.RelTypeProtos.SubRelType;

import eu.dnetlib.data.proto.TypeProtos.Type;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.rmi.MSROException;

public class DedupSimilarityToActionsJobNode extends DedupConfigurationAwareJobNode {

	private static final Log log = LogFactory.getLog(DedupSimilarityToActionsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Override

	protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {

		super.prepareJob(job, token);

		final String entityType = token.getEnv().getAttribute("entityType");

		if (StringUtils.isBlank(entityType))

			throw new MSROException("unable to find wf param: entityType");

		String cf = "_" + SubRelType.dedupSimilarity + "_" + DedupSimilarity.RelName.isSimilarTo;

		switch (Type.valueOf(entityType)) {

		case organization:

			cf = RelType.organizationOrganization + cf;

			break;

		case person:

			cf = RelType.personPerson + cf;

			break;

		case result:

			cf = RelType.resultResult + cf;

			break;

		default:

			throw new MSROException("invalid parameter entityType: " + entityType);

		}

		log.info("using similarity CF: " + cf);

		job.getParameters().put("similarityCF", cf);

		token.getEnv().setAttribute("similarityCF", cf);

	}

}

package eu.dnetlib.msro.workflows.dedup;

import java.io.StringReader;

import java.util.Iterator;

import java.util.List;

import java.util.Queue;

import javax.annotation.Resource;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.antlr.stringtemplate.StringTemplate;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Required;

import com.google.common.collect.Lists;

import com.google.common.collect.Queues;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.data.proto.TypeProtos.Type;

import eu.dnetlib.enabling.resultset.IterableResultSetFactory;

import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;

import eu.dnetlib.msro.workflows.hadoop.utils.Similarity;

import eu.dnetlib.msro.workflows.hadoop.utils.SimilarityMeshBuilder;

import eu.dnetlib.msro.workflows.nodes.AsyncJobNode;

public class BuildSimilarityMeshJobNode extends AsyncJobNode {

	private static final Log log = LogFactory.getLog(BuildSimilarityMeshJobNode.class);

	/** The result set factory. */

	@Resource(name = "iterableResultSetFactory")

	private IterableResultSetFactory resultSetFactory;

	/** The result set client factory. */

	@Resource(name = "resultSetClientFactory")

	private ResultSetClientFactory resultSetClientFactory;

	private StringTemplate similarity;

	private String inputEprParam;

	private String outputEprParam;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final String inputEpr = token.getEnv().getAttribute(getInputEprParam());

		final Iterator<String> rsClient = resultSetClientFactory.getClient(inputEpr).iterator();

		final Queue<Object> queue = Queues.newLinkedBlockingQueue();

		final SAXReader reader = new SAXReader();

		if (rsClient.hasNext()) {

			populateQueue(queue, reader, rsClient.next());

		}

		final W3CEndpointReference eprOut = resultSetFactory.createIterableResultSet(new Iterable<String>() {

			@Override

			public Iterator<String> iterator() {

				return new Iterator<String>() {

					@Override

					public boolean hasNext() {

						synchronized (queue) {

							return !queue.isEmpty();

						}

					}

					@Override

					public String next() {

						synchronized (queue) {

							final Object o = queue.poll();

							while (queue.isEmpty() && rsClient.hasNext()) {

								populateQueue(queue, reader, rsClient.next());

							}

							return buildSimilarity((Similarity) o);

						}

					}

					@Override

					public void remove() {

						throw new UnsupportedOperationException();

					}

				};

			}

		});

		token.getEnv().setAttribute(getOutputEprParam(), eprOut.toString());

		return Arc.DEFAULT_ARC;

	}

	private void populateQueue(final Queue<Object> q, final SAXReader r, final String xml) {

		try {

			final Document d = r.read(new StringReader(xml));

			final String groupid = d.valueOf("//FIELD[@name='id']");

			final List<?> items = d.selectNodes("//FIELD[@name='group']/ITEM");

			final String entitytype = d.valueOf("//FIELD[@name='entitytype']");

			final List<String> group = Lists.newArrayList();

			for (final Object id : items) {

				group.add(((Node) id).getText());

			}

			// compute the full mesh

			final Type type = Type.valueOf(entitytype);

			final List<Similarity> mesh = SimilarityMeshBuilder.build(type, group);

			// total += mesh.size();

			if (log.isDebugEnabled()) {

				log.debug(String.format("built mesh for group '%s', size %d", groupid, mesh.size()));

			}

			for (final Similarity s : mesh) {

				if (log.isDebugEnabled()) {

					log.debug(String.format("adding to queue: %s", s.toString()));

				}

				q.add(s);

			}

		} catch (final DocumentException e) {

			log.error("invalid document: " + xml);

		}

	}

	private String buildSimilarity(final Similarity s) {

		final StringTemplate template = new StringTemplate(getSimilarity().getTemplate());

		template.setAttribute("source", s.getPair().getKey());

		template.setAttribute("target", s.getPair().getValue());

		template.setAttribute("type", s.getType().toString());

		final String res = template.toString();

		return res;

	}

	public String getInputEprParam() {

		return inputEprParam;

	}

	public void setInputEprParam(final String inputEprParam) {

		this.inputEprParam = inputEprParam;

	}

	public String getOutputEprParam() {

		return outputEprParam;

	}

	public void setOutputEprParam(final String outputEprParam) {

		this.outputEprParam = outputEprParam;

	}

	public StringTemplate getSimilarity() {

		return similarity;

	}

	@Required

	public void setSimilarity(final StringTemplate similarity) {

		this.similarity = similarity;

	}

}

package eu.dnetlib.msro.workflows.dedup;

import org.apache.commons.lang.StringUtils;

import org.springframework.beans.factory.annotation.Autowired;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.msro.workflows.dedup.conf.DedupConfigurationOrchestration;

import eu.dnetlib.msro.workflows.dedup.conf.DedupConfigurationOrchestrationLoader;

import eu.dnetlib.msro.workflows.nodes.AsyncJobNode;

public class DedupConfigurationSetterJobNode extends AsyncJobNode {

	private String dedupConfigSequence;

	private String dedupConfigSequenceParam;

	@Autowired

	private DedupConfigurationOrchestrationLoader dedupOrchestrationLoader;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		if (StringUtils.isBlank(getDedupConfigSequence())) throw new IllegalArgumentException("missing configuration sequence");

		final DedupConfigurationOrchestration dedupOrchestration = dedupOrchestrationLoader.loadByActionSetId(getDedupConfigSequence());

		token.getEnv().setAttribute("entityType", dedupOrchestration.getEntity().getName());

		token.getEnv().setAttribute("entityTypeId", dedupOrchestration.getEntity().getCode());

		token.getEnv().setAttribute(getDedupConfigSequenceParam(), dedupOrchestration.toString());

		return Arc.DEFAULT_ARC;

	}

	public String getDedupConfigSequence() {

		return dedupConfigSequence;

	}

	public void setDedupConfigSequence(final String dedupConfigSequence) {

		this.dedupConfigSequence = dedupConfigSequence;

	}

	public String getDedupConfigSequenceParam() {

		return dedupConfigSequenceParam;

	}

	public void setDedupConfigSequenceParam(final String dedupConfigSequenceParam) {

		this.dedupConfigSequenceParam = dedupConfigSequenceParam;

	}

}

package eu.dnetlib.msro.workflows.hadoop.utils;

import java.util.Collection;

import java.util.Set;

import com.google.common.base.Function;

import com.google.common.base.Predicate;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.google.common.collect.Sets;

import eu.dnetlib.data.proto.DatasourceOrganizationProtos.DatasourceOrganization.Provision;

import eu.dnetlib.data.proto.DedupProtos.Dedup;

import eu.dnetlib.data.proto.DedupSimilarityProtos.DedupSimilarity;

import eu.dnetlib.data.proto.PersonPersonProtos.PersonPerson.CoAuthorship;

import eu.dnetlib.data.proto.PersonResultProtos.PersonResult.Authorship;

import eu.dnetlib.data.proto.ProjectOrganizationProtos.ProjectOrganization.Participation;

import eu.dnetlib.data.proto.ProjectPersonProtos.ProjectPerson.ContactPerson;

import eu.dnetlib.data.proto.RelTypeProtos.RelType;

import eu.dnetlib.data.proto.RelTypeProtos.SubRelType;

import eu.dnetlib.data.proto.ResultProjectProtos.ResultProject.Outcome;

import eu.dnetlib.data.proto.ResultResultProtos.ResultResult.PublicationDataset;

import eu.dnetlib.data.proto.ResultResultProtos.ResultResult.Similarity;

import eu.dnetlib.data.proto.TypeProtos.Type;

/**

 * Common static utility methods to manage the hbase tables

 *

 * @author claudio

 *

 */

public class HBaseTableUtils {

	private static final String _ = "_";

	private static Function<Type, String> typeName = new Function<Type, String>() {

		@Override

		public String apply(final Type type) {

			return type.toString();

		}

	};

	public static Set<String> listAllColumns() {

		final Set<String> union = Sets.union(listEntities(), listRelationships());

		return Sets.union(union, listDedupRelationships());

	}

	public static Set<String> listDedupColumns(final Collection<Type> entityTypes) {

		final Set<String> entities = listEntities(Lists.newArrayList(Iterables.transform(entityTypes, typeName)));

		return Sets.union(entities, listDedupRelationships());

	}

	private static Set<String> listDedupRelationships() {

		final Set<String> cfs = Sets.newHashSet();

		cfs.add(RelType.organizationOrganization + _ + SubRelType.dedup + _ + Dedup.RelName.merges);

		cfs.add(RelType.organizationOrganization + _ + SubRelType.dedup + _ + Dedup.RelName.isMergedIn);

		cfs.add(RelType.organizationOrganization + _ + SubRelType.dedupSimilarity + _ + DedupSimilarity.RelName.isSimilarTo);

		cfs.add(RelType.personPerson + _ + SubRelType.dedup + _ + Dedup.RelName.merges);

		cfs.add(RelType.personPerson + _ + SubRelType.dedup + _ + Dedup.RelName.isMergedIn);

		cfs.add(RelType.personPerson + _ + SubRelType.dedupSimilarity + _ + DedupSimilarity.RelName.isSimilarTo);

		cfs.add(RelType.resultResult + _ + SubRelType.dedup + _ + Dedup.RelName.merges);

		cfs.add(RelType.resultResult + _ + SubRelType.dedup + _ + Dedup.RelName.isMergedIn);

		cfs.add(RelType.resultResult + _ + SubRelType.dedupSimilarity + _ + DedupSimilarity.RelName.isSimilarTo);

		return cfs;

	}

	private static Set<String> listEntities(final Collection<String> entityType) {

		return Sets.newHashSet(Iterables.filter(Iterables.transform(Lists.newArrayList(Type.values()), typeName), new Predicate<String>() {

			@Override

			public boolean apply(final String s) {

				return entityType.contains(s);

			}

		}));

	}

	public static Set<String> listEntities() {

		return Sets.newHashSet(Iterables.transform(Lists.newArrayList(Type.values()), typeName));

	}

	public static Set<String> listRelationships() {

		final Set<String> cfs = Sets.newHashSet();

		cfs.add(RelType.datasourceOrganization + _ + SubRelType.provision + _ + Provision.RelName.isProvidedBy);

		cfs.add(RelType.datasourceOrganization + _ + SubRelType.provision + _ + Provision.RelName.provides);

		cfs.add(RelType.personPerson + _ + SubRelType.coauthorship + _ + CoAuthorship.RelName.isCoauthorOf);

		cfs.add(RelType.personResult + _ + SubRelType.authorship + _ + Authorship.RelName.isAuthorOf);

		cfs.add(RelType.personResult + _ + SubRelType.authorship + _ + Authorship.RelName.hasAuthor);

		cfs.add(RelType.projectOrganization + _ + SubRelType.participation + _ + Participation.RelName.hasParticipant);

		cfs.add(RelType.projectOrganization + _ + SubRelType.participation + _ + Participation.RelName.isParticipant);

		cfs.add(RelType.projectPerson + _ + SubRelType.contactPerson + _ + ContactPerson.RelName.isContact);

		cfs.add(RelType.projectPerson + _ + SubRelType.contactPerson + _ + ContactPerson.RelName.hasContact);

		cfs.add(RelType.resultProject + _ + SubRelType.outcome + _ + Outcome.RelName.isProducedBy);

		cfs.add(RelType.resultProject + _ + SubRelType.outcome + _ + Outcome.RelName.produces);

		cfs.add(RelType.resultResult + _ + SubRelType.similarity + _ + Similarity.RelName.hasAmongTopNSimilarDocuments);

		cfs.add(RelType.resultResult + _ + SubRelType.similarity + _ + Similarity.RelName.isAmongTopNSimilarDocuments);

		cfs.add(RelType.resultResult + _ + SubRelType.publicationDataset + _ + PublicationDataset.RelName.isRelatedTo);

		return cfs;

	}

}

package eu.dnetlib.msro.workflows.dedup.conf;

import java.util.Queue;

import com.google.gson.Gson;

import com.google.gson.GsonBuilder;

import eu.dnetlib.pace.config.DedupConfig;

/**

 * The Class DedupConfigurationOrchestration.

 */

public class DedupConfigurationOrchestration {

	/** The entity. */

	private Entity entity;

	/** The action set id. */

	private String actionSetId;

	/** The configurations. */

	private Queue<DedupConfig> configurations;

	public DedupConfigurationOrchestration() {}

	/**

	 * Instantiates a new dedup configuration orchestration.

	 *

	 * @param entity

	 *            the entity

	 * @param actionSetId

	 *            the action set id

	 * @param configurations

	 *            the configurations

	 */

	public DedupConfigurationOrchestration(final Entity entity, final String actionSetId, final Queue<DedupConfig> configurations) {

		super();

		this.setEntity(entity);

		this.setActionSetId(actionSetId);

		this.setConfigurations(configurations);

	}

	/**

	 * Gets the entity.

	 *

	 * @return the entity

	 */

	public Entity getEntity() {

		return entity;

	}

	/**

	 * Gets the action set id.

	 *

	 * @return the action set id

	 */

	public String getActionSetId() {

		return actionSetId;

	}

	/**

	 * Gets the configurations.

	 *

	 * @return the configurations

	 */

	public Queue<DedupConfig> getConfigurations() {

		return configurations;

	}

	public void setEntity(final Entity entity) {

		this.entity = entity;

	}

	public void setActionSetId(final String actionSetId) {

		this.actionSetId = actionSetId;

	}

	public void setConfigurations(final Queue<DedupConfig> configurations) {

		this.configurations = configurations;

	}

	/**

	 * From json.

	 *

	 * @param json

	 *            the json

	 * @return the dedup configuration orchestration

	 */

	public static DedupConfigurationOrchestration fromJSON(final String json) {

		return new Gson().fromJson(json, DedupConfigurationOrchestration.class);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.lang.Object#toString()

	 */

	@Override

	public String toString() {

		return new GsonBuilder().setPrettyPrinting().create().toJson(this);

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-deduplication</artifactId>

	<packaging>jar</packaging>

	<version>1.1.3</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-deduplication/tags/dnet-deduplication-1.1.3</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-msro-service</artifactId>

			<version>[3.0.0,4.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-hadoop-service-rmi</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-actionmanager-api</artifactId>

			<version>[3.0.0,4.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-modular-ui</artifactId>

			<version>[3.0.0,4.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-index-solr-client</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-openaireplus-mapping-utils</artifactId>

			<version>[3.0.0,4.0.0)</version>

		</dependency>

		<dependency>

			<groupId>javax.servlet</groupId>

			<artifactId>javax.servlet-api</artifactId>

			<version>${javax.servlet.version}</version>

			<scope>provided</scope>

		</dependency>

		<dependency>

			<groupId>com.fasterxml.jackson.core</groupId>

			<artifactId>jackson-databind</artifactId>

			<version>2.4.3</version>

		</dependency>

		<dependency>

			<groupId>com.google.guava</groupId>

			<artifactId>guava</artifactId>

			<version>${google.guava.version}</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

	</dependencies>

</project>

package eu.dnetlib.msro.workflows.hadoop.hbase;

import java.io.IOException;

import java.util.Map;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import com.googlecode.sarasvati.Engine;

import com.googlecode.sarasvati.NodeToken;

import com.googlecode.sarasvati.env.Env;

import eu.dnetlib.data.hadoop.rmi.HadoopBlackboardActions;

import eu.dnetlib.data.hadoop.rmi.HadoopService;

import eu.dnetlib.enabling.resultset.rmi.ResultSetException;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions;

import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;

import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;

import eu.dnetlib.msro.workflows.nodes.blackboard.BlackboardWorkflowJobListener;

import eu.dnetlib.msro.workflows.resultset.ProcessCountingResultSetFactory;

import eu.dnetlib.msro.workflows.util.ProgressProvider;

import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider;

import eu.dnetlib.msro.workflows.util.WorkflowsConstants;

public class DeleteHBaseRecordsJobNode extends BlackboardJobNode implements ProgressJobNode {

	private static final Log log = LogFactory.getLog(DeleteHBaseRecordsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	private String inputEprParam;

	private String hbaseTableProperty;

	private String cluster;

	private String xslt;

	private boolean simulation = false;

	private ProgressProvider progressProvider;

	private ProcessCountingResultSetFactory processCountingResultSetFactory;

	@Override

	protected String obtainServiceId(final NodeToken token) {

		return getServiceLocator().getServiceId(HadoopService.class);

	}

	@Override

	protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {

		log.info("Invoking blackboard method");

		job.setAction(HadoopBlackboardActions.DELETE_EPR_HBASE.toString());

		job.getParameters().put("input_epr", DnetXsltFunctions.encodeBase64(prepareEpr(token)));

		job.getParameters().put("xslt", DnetXsltFunctions.encodeBase64(prepareXslt()));

		job.getParameters().put("table", getPropertyFetcher().getProperty(getHbaseTableProperty()));

		job.getParameters().put("cluster", cluster);

		job.getParameters().put("simulation", String.valueOf(isSimulation()));

	}

	@Override

	protected BlackboardWorkflowJobListener generateBlackboardListener(final Engine engine, final NodeToken token) {

		return new BlackboardWorkflowJobListener(engine, token) {

			@Override

			protected void populateEnv(final Env env, final Map<String, String> responseParams) {

				final String count = responseParams.get("count");

				log.info(String.format("Deleted %s objects from HBase table %s, cluster %s", count, getPropertyFetcher().getProperty(getHbaseTableProperty()),

						getCluster()));

				env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + getName() + ":count", count);

			}

		};

	}

	private String prepareEpr(final NodeToken token) throws ResultSetException {

		final String epr = token.getEnv().getAttribute(inputEprParam);

		final ResultsetProgressProvider resultsetProgressProvider = processCountingResultSetFactory.createProgressProvider(token.getProcess(), epr);

		setProgressProvider(resultsetProgressProvider);

		return resultsetProgressProvider.getEpr().toString();

	}

	private String prepareXslt() throws IOException {

		return (xslt == null) || xslt.isEmpty() ? "" : IOUtils.toString(getClass().getResourceAsStream(xslt));

	}

	public String getInputEprParam() {

		return inputEprParam;

	}

	public void setInputEprParam(final String inputEprParam) {

		this.inputEprParam = inputEprParam;

	}

	public String getHbaseTableProperty() {

		return hbaseTableProperty;

	}

	public void setHbaseTableProperty(final String hbaseTableProperty) {

		this.hbaseTableProperty = hbaseTableProperty;

	}

	@Required

	public void setProcessCountingResultSetFactory(final ProcessCountingResultSetFactory processCountingResultSetFactory) {

		this.processCountingResultSetFactory = processCountingResultSetFactory;

	}

	@Override

	public ProgressProvider getProgressProvider() {

		return progressProvider;

	}

	public void setProgressProvider(final ProgressProvider progressProvider) {

		this.progressProvider = progressProvider;

	}

	public ProcessCountingResultSetFactory getProcessCountingResultSetFactory() {

		return processCountingResultSetFactory;

	}

	public String getXslt() {

		return xslt;

	}

	public void setXslt(final String xslt) {

		this.xslt = xslt;

	}

	public String getCluster() {

		return cluster;

	}

	public void setCluster(final String cluster) {

		this.cluster = cluster;

	}

	public boolean isSimulation() {

		return simulation;

	}

	public void setSimulation(final boolean simulation) {

		this.simulation = simulation;

	}

}

var dedupInspectorControllers = angular.module('dedupInspectorControllers', []);

function common_init($scope, $http, $sce, $location) {

	initSpinner();

	$scope.showError        = function(error)   { show_notification("error", error); }

	$scope.showNotification = function(message) { show_notification("info", message); }

	$scope.showSpinner      = function()        { showSpinner(); }

	$scope.hideSpinner      = function()        { hideSpinner(); }

	$scope.to_trusted       = function(html)    { return $sce.trustAsHtml(html); }

	$scope.go               = function(path)    { document.location.href='#'+path; }

	$scope.encodeValue      = function(val)     { return val; }

	$scope.encodeDateParam = function(date) {

		var year = date.getFullYear();

		var month = ("00" + (date.getMonth() + 1)).slice(-2);

		var day = ("00" + date.getDate()).slice(-2);

		return year + month + day;

	}

	$scope.decodeDateParam = function(s) {

		var year = s.substring(0,4);

		var month = s.substring(4,6);

		var day = s.substring(6,8);

		return new Date(year,month-1,day);

	}

}

dedupInspectorControllers.controller('addSimRelsCtrl', [

	'$scope', '$http', '$sce', '$location', '$routeParams', 'groupService',

	function ($scope, $http, $sce, $location, $routeParams, groupService) {

		common_init($scope, $http, $sce, $location);

		$scope.validentityTypes = { 

			'result' 		: 	{id:'50', type:'result', label:'Publication'}, 

			'organization' 	: 	{id:'20', type:'organization', label:'Organization'},

			'person' 		: 	{id:'30', type:'person', label:'Person'} 

		};

		$scope.group = groupService.getGroup();

		$scope.dissimilar = groupService.getDissimilar();

		if ($scope.group == null || $scope.group.entityType.type != $scope.validentityTypes[$routeParams.entityType].type) {

			$scope.group = {

				actionSet : $routeParams.actionSet,

				details : {},

				byRoot : {},

				byMerged : {},

				group : [],

				rootIds : [],

				detailList : [],

				rootList : [],

				entityType : $scope.validentityTypes[$routeParams.entityType]

			}

			groupService.setGroup($scope.group);

		};

		$scope.group.actionSet = $routeParams.actionSet;

		if($scope.dissimilar == null || $scope.group.entityType.type != $scope.validentityTypes[$routeParams.entityType].type) {

			$scope.dissimilar = {

				group : {},

				entityType : $scope.validentityTypes[$routeParams.entityType]

			};

			groupService.setDissimilar($scope.dissimilar);

		}

		$scope.groupIdDetails = {

			map : {},

			list : []

		};

		if($routeParams.query && $routeParams.query.length > 0) {

			$scope.query = $routeParams.query;

		}

		if($routeParams.start) {

			$scope.start = parseInt($routeParams.start);

		} else {

			$scope.start = 0;

		}

		$scope.rows = 20;

		$scope.search = function() {

			$scope.showSpinner();

			var q = '';

			if($scope.query.match('^[0-9][0-9]\|.{12}::[a-zA-Z0-9]{32}$')) {

				q = 'objidentifier exact "' + $scope.query + '"';

			} else {

				q = $scope.query;

			}

			$http.defaults.headers.post["Content-Type"] = "application/x-www-form-urlencoded; charset=UTF-8";

			$http.post('dedup/search.do', $.param({

				'entityType' : $scope.group.entityType.type,

				'query'  	: q,

				'actionSet' : $scope.group.actionSet,

				'start'	 	: $scope.start,

				'rows'	 	: $scope.rows,

				'fields' 	: $scope.defaultFields()

			})).success(function(res) {

            	if(res) {

            		$scope.results = res;

            		$scope.hideSpinner();

            	} else {

                	$scope.hideSpinner();

            		$scope.showError('Registration failed');

            	}

			}).error(function(err) {

				$scope.hideSpinner();

				$scope.showError('Registration failed: ' + err.message);

			});			

		}

		$scope.defaultFields = function() {

			switch($scope.group.entityType.type) { 

			  case "result": 

				  return "title,dateofacceptance,description";

			  case "organization": 

				  return "legalname,legalshortname,websiteurl";

			  case "person": 

				  return "fullname";

			  default: 

				  return "";

			}	

		}

		$scope.allFields = function() {

			switch($scope.group.entityType.type) { 

			  case "result": 

				  return "title,dateofacceptance,author,description";

			  case "organization": 

				  return "legalname,legalshortname,country,websiteurl";

			  default: 

				  return "";

			}	

		}		

		if($scope.query) {

			$scope.search();

		}

		$scope.searchById = function(id, type, fields, responseCallback) {

			$http.post('dedup/searchById.do', $.param({

				'entityType' : type,

				'objidentifier' : id,

				'fields' : fields

			})).success(function(res) {

	        	if(res) {

	        		responseCallback(id, res);

	        	} else {

	        		$scope.showError('Unable to query: ' + mId);

	        	}

			}).error(function(err) {

				$scope.showError('Unable to query id "' + mId + '", error: ' + err.message);

			});

		}		

		$scope.resetForm = function() {

			if (confirm('Reset group?')) {

				$scope.doResetForm();

			}

		}

		$scope.doResetForm = function() {

			$scope.group.group = [];

			$scope.group.rootIds = [];

			$scope.group.details = {};

			$scope.group.byRoot = {};

			$scope.group.byMerged = {};

			$scope.group.rootList = [];

			$scope.group.detailList = [];

			$scope.groupIdDetails.map = {};

			$scope.groupIdDetails.list = [];

			$scope.dissimilar = {

				group : {},

				entityType : $scope.validentityTypes[$routeParams.entityType]

			};

		}

		$scope.commit = function() {

			if (confirm('Commit actions?')) {

				$scope.showSpinner();

				var g = {

					actionSet : $scope.group.actionSet,

					entityType : $scope.group.entityType,

					group : $scope.group.group,

					rootIds : $scope.group.rootIds,

					dissimilar : $scope.dissimilar.group

				}

				$http.defaults.headers.post["Content-Type"] = "application/json; charset=UTF-8";

				$http.post('dedup/commit.do', g).success(function(res) {

	            	if(res) {

	            		$scope.showNotification('Commit done');

	            		$scope.search();

	            		$scope.hideSpinner();

	            	} else {

	                	$scope.hideSpinner();

	            		$scope.showError('Commit failed');

	            	}

				}).error(function(err) {

					$scope.hideSpinner();

					$scope.showError('Commit failed: ' + err.message);

				});

				$scope.doResetForm();

			}

		}

		$scope.removeFromResultList = function(ids, results) {

			var tmp = [];	

			angular.forEach(results, function(r) {

		    	if(jQuery.grep(ids, function(val, i) {

		    		  return val == r.id;

				}).length == 0) {

		    		tmp.push(r);

		    	};

		    });		    

		    return tmp;			

		}

		$scope.removeFromIdList = function(ids, list) {

			var tmp = [];	

			angular.forEach(list, function(id) {

		    	if(jQuery.grep(ids, function(val, i) {

		    		  return val == id;

				}).length == 0) {

		    		tmp.push(id);

		    	};

		    });		    

		    return tmp;			

		}			

		$scope.add = function(id, idList) {

			var size = $scope.group.group.length;

			var ids = idList.split(",");

		    angular.forEach(ids, function(id) {

		    	if(jQuery.grep($scope.group.group, function(val, i) {

		    		  return val == id;