D-Net

package eu.dnetlib.pace.model;

import java.util.List;

import java.util.Map;

import com.google.gson.Gson;

import eu.dnetlib.pace.clustering.Acronyms;

import eu.dnetlib.pace.clustering.Clustering;

import eu.dnetlib.pace.clustering.ClusteringFunction;

import eu.dnetlib.pace.clustering.NgramPairs;

import eu.dnetlib.pace.clustering.Ngrams;

import eu.dnetlib.pace.clustering.RandomClusteringFunction;

import eu.dnetlib.pace.clustering.SpaceTrimmingFieldValue;

import eu.dnetlib.pace.clustering.SuffixPrefix;

public class ClusteringDef {

	private Clustering name;

	private List<String> fields;

	private Map<String, Integer> params;

	public ClusteringDef() {}

	public Clustering getName() {

		return name;

	}

	public void setName(final Clustering name) {

		this.name = name;

	}

	public ClusteringFunction getClusteringFunction() {

		switch (getName()) {

		case acronyms:

			return new Acronyms(getParams());

		case ngrams:

			return new Ngrams(getParams());

		case ngrampairs:

			return new NgramPairs(getParams());

		case suffixprefix:

			return new SuffixPrefix(getParams());

		case spacetrimmingfieldvalue:

			return new SpaceTrimmingFieldValue(getParams());

		default:

			return new RandomClusteringFunction(getParams());

		}

	}

	public List<String> getFields() {

		return fields;

	}

	public void setFields(final List<String> fields) {

		this.fields = fields;

	}

	public Map<String, Integer> getParams() {

		return params;

	}

	public void setParams(final Map<String, Integer> params) {

		this.params = params;

	}

	@Override

	public String toString() {

		return new Gson().toJson(this);

	}

}

package eu.dnetlib.pace.model;

import java.util.Iterator;

import java.util.List;

import org.apache.commons.collections.iterators.SingletonIterator;

import eu.dnetlib.pace.config.Type;

/**

 * The Class FieldValueImpl.

 */

public class FieldValueImpl extends AbstractField implements FieldValue {

	/** The value. */

	private Object value = null;

	/**

	 * Instantiates a new field value impl.

	 */

	public FieldValueImpl() {}

	/**

	 * Instantiates a new field value impl.

	 * 

	 * @param type

	 *            the type

	 * @param name

	 *            the name

	 * @param value

	 *            the value

	 */

	public FieldValueImpl(final Type type, final String name, final Object value) {

		super(type, name);

		this.value = value;

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.model.Field#isEmpty()

	 */

	@Override

	public boolean isEmpty() {

		if (value == null) return false;

		switch (type) {

		case String:

			return value.toString().isEmpty();

		case List:

			List<?> list = (List<?>) value;

			return list.isEmpty() || ((FieldValueImpl) list.get(0)).isEmpty();

		default:

			return true;

		}

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.model.FieldValue#getValue()

	 */

	@Override

	public Object getValue() {

		return value;

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.model.FieldValue#setValue(java.lang.Object)

	 */

	@Override

	public void setValue(final Object value) {

		this.value = value;

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.model.Field#stringValue()

	 */

	@Override

	// @SuppressWarnings("unchecked")

	public String stringValue() {

		return String.valueOf(getValue());

		// switch (getType()) {

		//

		// case Int:

		// return String.valueOf(getValue());

		// case List:

		// return Joiner.on(" ").join((List<String>) getValue());

		// case String:

		// return (String) getValue();

		// default:

		// throw new IllegalArgumentException("Unknown type: " + getType().toString());

		// }

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.lang.Iterable#iterator()

	 */

	@Override

	@SuppressWarnings("unchecked")

	public Iterator<Field> iterator() {

		return new SingletonIterator(this);

	}

}

package eu.dnetlib.pace.model;

/**

 * The Interface FieldValue.

 */

public interface FieldValue extends Field {

	/**

	 * Gets the value.

	 * 

	 * @return the value

	 */

	public Object getValue();

	/**

	 * Sets the value.

	 * 

	 * @param value

	 *            the new value

	 */

	public void setValue(final Object value);

}

package eu.dnetlib.pace.model;

import java.lang.reflect.Type;

import com.google.gson.Gson;

import com.google.gson.GsonBuilder;

import com.google.gson.InstanceCreator;

import com.google.gson.JsonDeserializationContext;

import com.google.gson.JsonDeserializer;

import com.google.gson.JsonElement;

import com.google.gson.JsonParseException;

/**

 * The Class MapDocumentSerializer.

 */

public class MapDocumentSerializer implements InstanceCreator<MapDocument> {

	@Override

	public MapDocument createInstance(final Type type) {

		return new MapDocument();

	}

	/**

	 * Decode.

	 *

	 * @param bytes

	 *            the bytes

	 * @return the map document

	 */

	public static MapDocument decode(final byte[] bytes) {

		GsonBuilder gson = new GsonBuilder();

		gson.registerTypeAdapter(Field.class, new JsonDeserializer<Field>() {

			@Override

			public Field deserialize(final JsonElement json, final Type typeOfT, final JsonDeserializationContext context) throws JsonParseException {

				FieldListImpl fl = new FieldListImpl();

				if (json.isJsonObject()) {

					String name = json.getAsJsonObject().get("name").getAsString();

					String type = json.getAsJsonObject().get("type").getAsString();

					String value = json.getAsJsonObject().get("value").getAsString();

					fl.add(new FieldValueImpl(eu.dnetlib.pace.config.Type.valueOf(type), name, value));

				}

				return fl;

			}

		});

		return gson.create().fromJson(new String(bytes), MapDocument.class);

	}

	/**

	 * To string.

	 *

	 * @param doc

	 *            the doc

	 * @return the string

	 */

	public static String toString(final MapDocument doc) {

		return new Gson().toJson(doc);

	}

	/**

	 * To byte array.

	 *

	 * @param doc

	 *            the doc

	 * @return the byte[]

	 */

	public static byte[] toByteArray(final MapDocument doc) {

		return toString(doc).getBytes();

	}

}

package eu.dnetlib.pace.model;

import java.text.Normalizer;

import java.util.List;

import java.util.Set;

import com.google.common.base.Joiner;

import com.google.common.base.Splitter;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.google.common.hash.Hashing;

import eu.dnetlib.pace.clustering.NGramUtils;

import eu.dnetlib.pace.util.Capitalise;

import eu.dnetlib.pace.util.DotAbbreviations;

public class Person {

	private List<String> name = Lists.newArrayList();

	private List<String> surname = Lists.newArrayList();

	private List<String> fullname = Lists.newArrayList();

	private final String original;

	private static Set<String> particles = null;

	public Person(String s, boolean aggressive) {

		original = s;

		s = Normalizer.normalize(s, Normalizer.Form.NFD);

		s = s.replaceAll("\\(.+\\)", "");

		s = s.replaceAll("\\[.+\\]", "");

		s = s.replaceAll("\\{.+\\}", "");

		s = s.replaceAll("\\s+-\\s+", "-");

		s = s.replaceAll("[\\p{Punct}&&[^,-]]", " ");

		s = s.replaceAll("\\d", " ");

		s = s.replaceAll("\\n", " ");

		s = s.replaceAll("\\.", " ");

		s = s.replaceAll("\\s+", " ");

		if (aggressive) {

			s = s.replaceAll("[\\p{InCombiningDiacriticalMarks}&&[^,-]]", "");

			//s = s.replaceAll("[\\W&&[^,-]]", "");

		}

		if (s.contains(",")) {

			String[] arr = s.split(",");

			if (arr.length == 1) {

				fullname = splitTerms(arr[0]);

			} else if (arr.length > 1) {

				surname = splitTerms(arr[0]);

				name = splitTerms(arr[1]);

				fullname.addAll(surname);

				fullname.addAll(name);

			}

		} else {

			fullname = splitTerms(s);

			int lastInitialPosition = fullname.size();

			boolean hasSurnameInUpperCase = false;

			for (int i = 0; i < fullname.size(); i++) {

				String term = fullname.get(i);

				if (term.length() == 1) {

					lastInitialPosition = i;

				} else if (term.equals(term.toUpperCase())) {

					hasSurnameInUpperCase = true;

				}

			}

			if (lastInitialPosition < fullname.size() - 1) { // Case: Michele G. Artini

				name = fullname.subList(0, lastInitialPosition + 1);

				surname = fullname.subList(lastInitialPosition + 1, fullname.size());

			} else if (hasSurnameInUpperCase) { // Case: Michele ARTINI

				for (String term : fullname) {

					if (term.length() > 1 && term.equals(term.toUpperCase())) {

						surname.add(term);

					} else {

						name.add(term);

					}

				}

			}

		}

	}

	private List<String> splitTerms(String s) {

		if (particles == null) {

			particles = NGramUtils.loadFromClasspath("/eu/dnetlib/pace/config/name_particles.txt");

		}

		List<String> list = Lists.newArrayList();

		for (String part : Splitter.on(" ").omitEmptyStrings().split(s)) {

			if (!particles.contains(part.toLowerCase())) {

				list.add(part);

			}

		}

		return list;

	}

	public List<String> getName() {

		return name;

	}

	public List<String> getSurname() {

		return surname;

	}

	public List<String> getFullname() {

		return fullname;

	}

	public String getOriginal() {

		return original;

	}

	public String hash() {

		return Hashing.murmur3_128().hashString(getNormalisedFullname()).toString();

	}

	public String getNormalisedFirstName() {

		return Joiner.on(" ").join(getCapitalFirstnames());

	}

	public String getNormalisedSurname() {

		return Joiner.on(" ").join(getCapitalSurname());

	}

	public String getNormalisedFullname() {

		return isAccurate() ? getNormalisedSurname() + ", " + getNormalisedFirstName() : Joiner.on(" ").join(fullname);

	}

	public List<String> getCapitalFirstnames() {

		return Lists.newArrayList(Iterables.transform(getNameWithAbbreviations(), new Capitalise()));

	}

	public List<String> getCapitalSurname() {

		return Lists.newArrayList(Iterables.transform(surname, new Capitalise()));

	}

	public List<String> getNameWithAbbreviations() {

		return Lists.newArrayList(Iterables.transform(name, new DotAbbreviations()));

	}

	public boolean isAccurate() {

		return (name != null && surname != null && !name.isEmpty() && !surname.isEmpty());

	}

}

package eu.dnetlib.pace.model;

import java.util.Map;

import java.util.Set;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

/**

 * The Class MapDocument.

 */

public class MapDocument implements Document {

	/** The identifier. */

	private String identifier;

	/** The field map. */

	private Map<String, FieldListImpl> fieldMap;

	/**

	 * Instantiates a new map document.

	 */

	public MapDocument() {

		identifier = null;

		fieldMap = Maps.newHashMap();

	}

	/**

	 * Instantiates a new map document.

	 *

	 * @param identifier

	 *            the identifier

	 * @param fieldMap

	 *            the field map

	 */

	public MapDocument(final String identifier, final Map<String, FieldListImpl> fieldMap) {

		this.setIdentifier(identifier);

		this.fieldMap = fieldMap;

	}

	/**

	 * Instantiates a new map document.

	 *

	 * @param identifier

	 *            the identifier

	 * @param data

	 *            the data

	 */

	public MapDocument(final String identifier, final byte[] data) {

		MapDocument doc = MapDocumentSerializer.decode(data);

		this.fieldMap = doc.fieldMap;

		this.identifier = doc.identifier;

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.pace.model.document.Document#fields()

	 */

	@Override

	public Iterable<Field> fields() {

		return Lists.newArrayList(Iterables.concat(fieldMap.values()));

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.pace.model.document.Document#values(java.lang.String)

	 */

	@Override

	public FieldList values(final String name) {

		return fieldMap.get(name);

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.pace.model.document.Document#fieldNames()

	 */

	@Override

	public Set<String> fieldNames() {

		return fieldMap.keySet();

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see java.lang.Object#toString()

	 */

	@Override

	public String toString() {

		return MapDocumentSerializer.toString(this);

		// return String.format("Document(%s)", fieldMap.toString());

	}

	/**

	 * To byte array.

	 *

	 * @return the byte[]

	 */

	public byte[] toByteArray() {

		return MapDocumentSerializer.toByteArray(this);

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.pace.model.document.Document#getIdentifier()

	 */

	@Override

	public String getIdentifier() {

		return identifier;

	}

	/**

	 * Sets the identifier.

	 *

	 * @param identifier

	 *            the new identifier

	 */

	public void setIdentifier(final String identifier) {

		this.identifier = identifier;

	}

	/**

	 * Gets the field map.

	 *

	 * @return the field map

	 */

	public Map<String, FieldListImpl> getFieldMap() {

		return fieldMap;

	}

	/**

	 * Sets the field map.

	 *

	 * @param fieldMap

	 *            the field map

	 */

	public void setFieldMap(final Map<String, FieldListImpl> fieldMap) {

		this.fieldMap = fieldMap;

	}

}

package eu.dnetlib.pace.config;

import java.util.List;

import java.util.Map;

import eu.dnetlib.pace.condition.ConditionAlgo;

import eu.dnetlib.pace.model.ClusteringDef;

import eu.dnetlib.pace.model.FieldDef;

/**

 * Interface for PACE configuration bean.

 *

 * @author claudio

 */

public interface Config {

	/**

	 * Field configuration definitions.

	 *

	 * @return the list of definitions

	 */

	public List<FieldDef> model();

	/**

	 * Strict Pre-Condition definitions.

	 *

	 * @return the list of conditions

	 */

	public List<ConditionAlgo> strictConditions();

	/**

	 * Pre-Condition definitions.

	 *

	 * @return the list of conditions

	 */

	public List<ConditionAlgo> conditions();

	/**

	 * Clusterings.

	 *

	 * @return the list

	 */

	public List<ClusteringDef> clusterings();

	/**

	 * Blacklists.

	 *

	 * @return the map

	 */

	public Map<String, List<String>> blacklists();

}

package eu.dnetlib.pace.model;

import java.util.List;

/**

 * The Interface FieldList.

 */

public interface FieldList extends List<Field>, Field {

	/**

	 * String list.

	 * 

	 * @return the list

	 */

	public List<String> stringList();

}

package eu.dnetlib.pace.config;

/**

 * Enumerates the distance Algos.

 */

public enum Algo {

	/** The Jaro winkler. */

	JaroWinkler,

	/** The Jaro winkler title. */

	JaroWinklerTitle,

	/** The Levenstein. */

	Levenstein,

	/** The Level2 jaro winkler. */

	Level2JaroWinkler,

	/** The Level2 levenstein. */

	Level2Levenstein,

	/** The Sub string levenstein. */

	SubStringLevenstein,

	/** The Year levenstein. */

	YearLevenstein,

	/** The Sorted jaro winkler. */

	SortedJaroWinkler,

	/** The Sorted level2 jaro winkler. */

	SortedLevel2JaroWinkler,

	/** The Null. */

	Null

}

package eu.dnetlib.pace.model;

import java.util.Map;

/**

 * The Class DocumentBuilder.

 */

public class DocumentBuilder {

	/**

	 * New instance.

	 *

	 * @param identifier

	 *            the identifier

	 * @param fieldMap

	 *            the field map

	 * @return the map document

	 */

	public static MapDocument newInstance(final String identifier, final Map<String, FieldListImpl> fieldMap) {

		return new MapDocument(identifier, fieldMap);

	}

	/**

	 * New instance.

	 *

	 * @param identifier

	 *            the identifier

	 * @param fieldMap

	 *            the field map

	 * @return the map document

	 */

	public static MapDocument newInstance(final String identifier, final byte[] fieldMap) {

		return new MapDocument(identifier, fieldMap);

	}

}

package eu.dnetlib.pace.distance;

import java.util.List;

import eu.dnetlib.pace.condition.ConditionAlgo;

import eu.dnetlib.pace.model.Document;

import eu.dnetlib.pace.model.Field;

import eu.dnetlib.pace.model.FieldDef;

/**

 * The distance between two documents is given by the weighted mean of the field distances

 */

public class DistanceScorer {

	private List<FieldDef> fields;

	private List<ConditionAlgo> strictConditions;

	private List<ConditionAlgo> conditions;

	public DistanceScorer(final List<FieldDef> fields, final List<ConditionAlgo> strictConditions, final List<ConditionAlgo> conditions) {

		this.fields = fields;

		this.conditions = conditions;

		this.strictConditions = strictConditions;

	}

	public double distance(final Document a, final Document b) {

		double w = sumWeights(fields);

		double sum = 0.0;

		final int cond = verify(a, b, strictConditions, true);

		if (cond > 0) return 1.0;

		if (cond < 0) return 0.0;

		if (verify(a, b, conditions, true) >= 0) {

			for (final FieldDef fd : fields) {

				final double d = fieldDistance(a, b, fd);

				if (d > 0) {

					sum += d;

				} else {

					w -= fd.getWeight();

				}

			}

			return w == 0 ? 0 : sum / w;

		}

		return 0.0;

	}

	private int verify(final Document a, final Document b, final List<ConditionAlgo> conditions, final boolean strict) {

		int cond = 0;

		for (final ConditionAlgo cd : conditions) {

			final int verify = cd.verify(a, b);

			if (strict && (verify < 0)) return -1;

			cond += verify;

		}

		return cond;

	}

	private double fieldDistance(final Document a, final Document b, final FieldDef fd) {

		final double w = fd.getWeight();

		if ((w == 0)) return 0.0; // optimization for 0 weight

		else {

			final Field va = getValue(a, fd);

			final Field vb = getValue(b, fd);

			if (va.isEmpty() || vb.isEmpty()) {

				if (fd.isIgnoreMissing()) return -1;

				else return w;

			} else {

				if (va.getType().equals(vb.getType())) {

					final double d = fd.getDistanceAlgo().distance(va, vb);

					return w * d;

				}

				throw new IllegalArgumentException("Types are differents type");

			}

		}

	}

	private Field getValue(final Document d, final FieldDef fd) {

		return d.values(fd.getName());

	}

	private double sumWeights(final List<FieldDef> fields) {

		double sum = 0.0;

		for (final FieldDef fd : fields) {

			sum += fd.getWeight();

		}

		return sum;

	}

}

package eu.dnetlib.pace.config;

import java.util.HashSet;

import java.util.List;

import java.util.Set;

import com.google.common.collect.Lists;

import com.google.common.collect.Sets;

import com.google.gson.GsonBuilder;

public class WfConfig {

	/**

	 * Entity type.

	 */

	private String entityType = "";

	/**

	 * Field name used to sort the values in the reducer phase.

	 */

	private String orderField = "";

	/**

	 * Column Families involved in the relations redirection.

	 */

	private List<String> rootBuilder = Lists.newArrayList();

	/**

	 * Set of datasource namespace prefixes that won't be deduplicated.

	 */

	private Set<String> skipList = Sets.newHashSet();

	/**

	 * Subprefix used to build the root id, allows multiple dedup runs.

	 */

	private String dedupRun = "";

	/**

	 * Similarity threshold.

	 */

	private double threshold = 0;

	/** The queue max size. */

	private int queueMaxSize = 2000;

	/** The group max size. */

	private int groupMaxSize;

	/** The sliding window size. */

	private int slidingWindowSize;

	/** The configuration id. */

	private String configurationId;

	/** The include children. */

	private boolean includeChildren;

	/** Default maximum number of allowed children. */

	private final static int MAX_CHILDREN = 50;

	/** Maximum number of allowed children. */

	private int maxChildren = MAX_CHILDREN;

	public WfConfig() {}

	/**

	 * Instantiates a new dedup config.

	 *

	 * @param entityType

	 *            the entity type

	 * @param orderField

	 *            the order field

	 * @param rootBuilder

	 *            the root builder families

	 * @param dedupRun

	 *            the dedup run

	 * @param configurationId

	 *            the configuration identifier

	 * @param threshold

	 *            the threshold

	 * @param skipList

	 *            the skip list

	 * @param queueMaxSize

	 *            the queue max size

	 * @param groupMaxSize

	 *            the group max size

	 * @param slidingWindowSize

	 *            the sliding window size

	 * @param includeChildren

	 *            allows the children to be included in the representative records or not.

	 */

	public WfConfig(final String entityType, final String orderField, final List<String> rootBuilder, final String dedupRun,

			final double threshold,

			final Set<String> skipList, final int queueMaxSize, final int groupMaxSize, final int slidingWindowSize, final boolean includeChildren) {

		super();

		this.entityType = entityType;

		this.orderField = orderField;

		this.rootBuilder = rootBuilder;

		this.dedupRun = cleanupStringNumber(dedupRun);

		this.threshold = threshold;

		this.skipList = skipList;

		this.queueMaxSize = queueMaxSize;

		this.groupMaxSize = groupMaxSize;

		this.slidingWindowSize = slidingWindowSize;

		this.includeChildren = includeChildren;

	}

	/**

	 * Cleanup string number.

	 *

	 * @param s

	 *            the s

	 * @return the string

	 */

	private String cleanupStringNumber(final String s) {

		return s.contains("'") ? s.replaceAll("'", "") : s;

	}

	public String getEntityType() {

		return entityType;

	}

	public void setEntityType(final String entityType) {

		this.entityType = entityType;

	}

	public String getOrderField() {

		return orderField;

	}

	public void setOrderField(final String orderField) {

		this.orderField = orderField;

	}

	public List<String> getRootBuilder() {

		return rootBuilder;

	}

	public void setRootBuilder(final List<String> rootBuilder) {

		this.rootBuilder = rootBuilder;

	}

	public Set<String> getSkipList() {

		return skipList != null ? skipList : new HashSet<String>();

	}

	public void setSkipList(final Set<String> skipList) {

		this.skipList = skipList;

	}

	public String getDedupRun() {

		return dedupRun;

	}

	public void setDedupRun(final String dedupRun) {

		this.dedupRun = dedupRun;

	}

	public double getThreshold() {

		return threshold;

	}

	public void setThreshold(final double threshold) {

		this.threshold = threshold;

	}

	public int getQueueMaxSize() {

		return queueMaxSize;

	}

	public void setQueueMaxSize(final int queueMaxSize) {

		this.queueMaxSize = queueMaxSize;

	}

	public int getGroupMaxSize() {

		return groupMaxSize;

	}

	public void setGroupMaxSize(final int groupMaxSize) {

		this.groupMaxSize = groupMaxSize;

	}

	public int getSlidingWindowSize() {

		return slidingWindowSize;

	}

	public void setSlidingWindowSize(final int slidingWindowSize) {

		this.slidingWindowSize = slidingWindowSize;

	}

	public String getConfigurationId() {

		return configurationId;

	}

	public void setConfigurationId(final String configurationId) {

		this.configurationId = configurationId;

	}

	public boolean isIncludeChildren() {

		return includeChildren;

	}

	public void setIncludeChildren(final boolean includeChildren) {

		this.includeChildren = includeChildren;

	}

	public int getMaxChildren() {

		return maxChildren;

	}

	public void setMaxChildren(final int maxChildren) {

		this.maxChildren = maxChildren;

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see java.lang.Object#toString()

	 */

	@Override

	public String toString() {

		return new GsonBuilder().setPrettyPrinting().create().toJson(this);

	}

}

package eu.dnetlib.pace.distance;

import com.wcohen.ss.AbstractStringDistance;

/**

 * The Class SortedJaroWinkler.

 */

public class SortedJaroWinkler extends SortedSecondStringDistanceAlgo {

	/**

	 * Instantiates a new sorted jaro winkler.

	 * 

	 * @param weight

	 *            the weight

	 */

	public SortedJaroWinkler(final double weight) {

		super(weight, new com.wcohen.ss.JaroWinkler());

	}

	/**

	 * Instantiates a new sorted jaro winkler.

	 * 

	 * @param weight

	 *            the weight

	 * @param ssalgo

	 *            the ssalgo

	 */

	protected SortedJaroWinkler(final double weight, final AbstractStringDistance ssalgo) {

		super(weight, ssalgo);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.distance.DistanceAlgo#getWeight()

	 */

	@Override

	public double getWeight() {

		return super.weight;

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.distance.SecondStringDistanceAlgo#normalize(double)

	 */

	@Override

	protected double normalize(final double d) {

		return d;

	}

}

package eu.dnetlib.pace.config;

import java.util.List;

import java.util.Map;

import org.apache.commons.collections.CollectionUtils;

import com.google.common.base.Predicate;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import eu.dnetlib.pace.condition.ConditionAlgo;

import eu.dnetlib.pace.model.ClusteringDef;

import eu.dnetlib.pace.model.CondDef;

import eu.dnetlib.pace.model.FieldDef;

public class PaceConfig {

	private List<FieldDef> model;

	private List<CondDef> strictConditions;

	private List<CondDef> conditions;

	private List<ClusteringDef> clustering;

	private Map<String, List<String>> blacklists;

	public PaceConfig() {}

	public List<FieldDef> getModel() {

		return model;

	}

	public void setModel(final List<FieldDef> fields) {

		this.model = fields;

	}

	public List<CondDef> getStrictConditions() {

		return strictConditions;

	}

	public void setStrictConditions(final List<CondDef> strictConditions) {

		this.strictConditions = strictConditions;

	}

	public List<CondDef> getConditions() {

		return conditions;

	}

	public List<ConditionAlgo> getConditionAlgos() {

		return asConditionAlgos(getConditions());

	}

	public List<ConditionAlgo> getStrictConditionAlgos() {

		return asConditionAlgos(getStrictConditions());

	}

	public void setConditions(final List<CondDef> conditions) {

		this.conditions = conditions;

	}

	public List<ClusteringDef> getClustering() {

		return clustering;

	}

	public void setClustering(final List<ClusteringDef> clustering) {

		this.clustering = clustering;

	}

	public Map<String, List<String>> getBlacklists() {

		return blacklists;

	}

	public void setBlacklists(final Map<String, List<String>> blacklists) {

		this.blacklists = blacklists;

	}

	// helper

	private List<ConditionAlgo> asConditionAlgos(final List<CondDef> defs) {

		final List<ConditionAlgo> algos = Lists.newArrayList();

		if (CollectionUtils.isEmpty(defs)) return algos;

		for (final CondDef cd : defs) {

			final List<FieldDef> fields = Lists.newArrayList(Iterables.filter(getModel(), new Predicate<FieldDef>() {

				@Override

				public boolean apply(final FieldDef fd) {

					return cd.getFields().contains(fd.getName());

				}

			}));

			algos.add(cd.getConditionAlgo(fields));

		}

		return algos;

	}

}

package eu.dnetlib.pace.distance;

public class LevensteinDate extends Levenstein {

	public LevensteinDate(double w) {

		super(w);

	}

	@Override

	public double distance(String a, String b) {

		return 1.0;

	}

	@Override

	public double getWeight() {

		return super.weight;

	}

}

package eu.dnetlib.pace.distance;

import org.apache.commons.lang.StringUtils;

import com.wcohen.ss.AbstractStringDistance;

import eu.dnetlib.pace.config.Type;

import eu.dnetlib.pace.model.Field;

/**

 * The Class SubStringLevenstein.

 */

public class SubStringLevenstein extends SecondStringDistanceAlgo {

	/** The limit. */

	protected int limit;

	/**

	 * Instantiates a new sub string levenstein.

	 * 

	 * @param w

	 *            the w

	 */

	public SubStringLevenstein(final double w) {

		super(w, new com.wcohen.ss.Levenstein());

	}

	/**

	 * Instantiates a new sub string levenstein.

	 * 

	 * @param w

	 *            the w

	 * @param limit

	 *            the limit

	 */

	public SubStringLevenstein(final double w, final int limit) {

		super(w, new com.wcohen.ss.Levenstein());

		this.limit = limit;

	}

	/**

	 * Instantiates a new sub string levenstein.

	 * 

	 * @param w

	 *            the w

	 * @param limit

	 *            the limit

	 * @param ssalgo

	 *            the ssalgo

	 */

	protected SubStringLevenstein(final double w, final int limit, final AbstractStringDistance ssalgo) {

		super(w, ssalgo);

		this.limit = limit;

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.distance.SecondStringDistanceAlgo#distance(eu.dnetlib.pace.model.Field, eu.dnetlib.pace.model.Field)

	 */

	@Override

	public double distance(final Field a, final Field b) {

		if (a.getType().equals(Type.String) && b.getType().equals(Type.String))

			return distance(StringUtils.left(a.stringValue(), limit), StringUtils.left(b.stringValue(), limit));

		throw new IllegalArgumentException("invalid types\n- A: " + a.toString() + "\n- B: " + b.toString());

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.distance.DistanceAlgo#getWeight()

	 */

	@Override

	public double getWeight() {

		return super.weight;

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.distance.SecondStringDistanceAlgo#normalize(double)

	 */

	@Override

	protected double normalize(final double d) {

		return 1 / Math.pow(Math.abs(d) + 1, 0.1);

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet-hadoop-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-pace-core</artifactId>

	<packaging>jar</packaging>

	<version>2.0.0</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-pace-core/tags/dnet-pace-core-2.0.0</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>edu.cmu</groupId>

			<artifactId>secondstring</artifactId>

			<version>1.0.0</version>

		</dependency>

		<dependency>

			<groupId>com.google.guava</groupId>

			<artifactId>guava</artifactId>

			<version>${google.guava.version}</version>

		</dependency>

		<dependency>

			<groupId>com.google.code.gson</groupId>

			<artifactId>gson</artifactId>

			<version>${google.gson.version}</version>

		</dependency>

		<dependency>

			<groupId>commons-lang</groupId>

			<artifactId>commons-lang</artifactId>

			<version>${commons.lang.version}</version>

		</dependency>

		<dependency>

			<groupId>commons-io</groupId>

			<artifactId>commons-io</artifactId>

			<version>${commons.io.version}</version>

		</dependency>

		<dependency>

			<groupId>commons-collections</groupId>

			<artifactId>commons-collections</artifactId>

			<version>${commons.collections.version}</version>

		</dependency>

		<dependency>

			<groupId>org.antlr</groupId>

			<artifactId>stringtemplate</artifactId>

			<version>3.2</version>

		</dependency>	

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

	</dependencies>

</project>

package eu.dnetlib.pace.distance;

import eu.dnetlib.pace.model.Document;

public class PaceDocumentDistance extends AbstractDistance<Document> {

	@Override

	protected Document toDocument(Document a) {

		return a;

	}

}

package eu.dnetlib.pace.config;

import static org.junit.Assert.assertNotNull;

import java.io.IOException;

import org.junit.Test;

import eu.dnetlib.pace.AbstractPaceTest;

public class ConfigTest extends AbstractPaceTest {

	@Test

	public void test() throws IOException {

		final DedupConfig cfg = DedupConfig.load(readFromClasspath("result.pace.conf.json"));

		assertNotNull(cfg);

		System.out.println(cfg);

	}

}