D-Net

package eu.dnetlib.pace.distance;

public class LevensteinDate extends Levenstein {

	public LevensteinDate(double w) {

		super(w);

	}

	@Override

	public double distance(String a, String b) {

		return 1.0;

	}

	@Override

	public double getWeight() {

		return super.weight;

	}

}

package eu.dnetlib.pace.distance;

import com.wcohen.ss.AbstractStringDistance;

//case class JaroWinkler(w: Double) extends SecondStringDistanceAlgo(w, new com.wcohen.ss.JaroWinkler())

public class JaroWinklerTitle extends SecondStringDistanceAlgo {

	public JaroWinklerTitle(double weight) {

		super(weight, new com.wcohen.ss.JaroWinkler());

	}

	protected JaroWinklerTitle(double weight, AbstractStringDistance ssalgo) {

		super(weight, ssalgo);

	}

	@Override

	public double distance(String a, String b) {

		String ca = cleanup(a);

		String cb = cleanup(b);

		boolean check = checkNumbers(ca, cb);

		return check ? 0.5 : normalize(ssalgo.score(ca, cb));

	}	

	@Override

	public double getWeight() {

		return super.weight;

	}

	@Override

	protected double normalize(double d) {

		return d;

	}

}

package eu.dnetlib.pace.distance;

import com.wcohen.ss.AbstractStringDistance;

public class Level2JaroWinklerTitle extends SecondStringDistanceAlgo {

	public Level2JaroWinklerTitle(final double w) {

		super(w, new com.wcohen.ss.Level2JaroWinkler());

	}

	protected Level2JaroWinklerTitle(final double w, final AbstractStringDistance ssalgo) {

		super(w, ssalgo);

	}

	@Override

	public double distance(final String a, final String b) {

		final String ca = cleanup(a);

		final String cb = cleanup(b);

		final boolean check = checkNumbers(ca, cb);

		if (check) return 0.5;

		final String cca = finalCleanup(ca);

		final String ccb = finalCleanup(cb);

		return ssalgo.score(cca, ccb);

	}

	@Override

	public double getWeight() {

		return super.weight;

	}

	@Override

	protected double normalize(final double d) {

		return d;

	}

}

package eu.dnetlib.pace.distance;

import eu.dnetlib.pace.model.Field;

/**

 * Each field is configured with a distance algo which knows how to compute the distance (0-1) between the fields of two

 * objects.

 */

public interface DistanceAlgo {

	public abstract double distance(Field a, Field b);

	public double getWeight();

}

package eu.dnetlib.pace.distance;

import java.util.List;

import eu.dnetlib.pace.condition.ConditionAlgo;

import eu.dnetlib.pace.model.Document;

import eu.dnetlib.pace.model.Field;

import eu.dnetlib.pace.model.FieldDef;

/**

 * The distance between two documents is given by the weighted mean of the field distances

 */

public class DistanceScorer {

	private List<FieldDef> fields;

	private List<ConditionAlgo> strictConditions;

	private List<ConditionAlgo> conditions;

	public DistanceScorer(final List<FieldDef> fields, final List<ConditionAlgo> strictConditions, final List<ConditionAlgo> conditions) {

		this.fields = fields;

		this.conditions = conditions;

		this.strictConditions = strictConditions;

	}

	public double distance(final Document a, final Document b) {

		double w = sumWeights(fields);

		double sum = 0.0;

		final int cond = verify(a, b, strictConditions, true);

		if (cond > 0) return 1.0;

		if (cond < 0) return 0.0;

		if (verify(a, b, conditions, true) >= 0) {

			for (final FieldDef fd : fields) {

				final double d = fieldDistance(a, b, fd);

				if (d > 0) {

					sum += d;

				} else {

					w -= fd.getWeight();

				}

			}

			return w == 0 ? 0 : sum / w;

		}

		return 0.0;

	}

	private int verify(final Document a, final Document b, final List<ConditionAlgo> conditions, final boolean strict) {

		int cond = 0;

		for (final ConditionAlgo cd : conditions) {

			final int verify = cd.verify(a, b);

			if (strict && (verify < 0)) return -1;

			cond += verify;

		}

		return cond;

	}

	private double fieldDistance(final Document a, final Document b, final FieldDef fd) {

		final double w = fd.getWeight();

		if ((w == 0)) return 0.0; // optimization for 0 weight

		else {

			final Field va = getValue(a, fd);

			final Field vb = getValue(b, fd);

			if (va.isEmpty() || vb.isEmpty()) {

				if (fd.isIgnoreMissing()) return -1;

				else return w;

			} else {

				if (va.getType().equals(vb.getType())) {

					final double d = fd.getDistanceAlgo().distance(va, vb);

					return w * d;

				}

				throw new IllegalArgumentException("Types are differents type");

			}

		}

	}

	private Field getValue(final Document d, final FieldDef fd) {

		return d.values(fd.getName());

	}

	private double sumWeights(final List<FieldDef> fields) {

		double sum = 0.0;

		for (final FieldDef fd : fields) {

			sum += fd.getWeight();

		}

		return sum;

	}

}

package eu.dnetlib.pace.distance;

import eu.dnetlib.pace.config.Config;

public interface Distance<A> {

	public double between(A a, A b, Config config);

}

package eu.dnetlib.pace.distance;

import java.util.Collections;

import java.util.List;

import com.google.common.collect.Lists;

import com.wcohen.ss.AbstractStringDistance;

import eu.dnetlib.pace.model.Field;

import eu.dnetlib.pace.model.FieldList;

/**

 * For the rest of the fields delegate the distance measure to the second string library.

 */

public abstract class SortedSecondStringDistanceAlgo extends SecondStringDistanceAlgo {

	/**

	 * Instantiates a new sorted second string distance algo.

	 * 

	 * @param weight

	 *            the weight

	 * @param ssalgo

	 *            the ssalgo

	 */

	protected SortedSecondStringDistanceAlgo(final double weight, final AbstractStringDistance ssalgo) {

		super(weight, ssalgo);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.distance.SecondStringDistanceAlgo#toList(eu.dnetlib.pace.model.Field)

	 */

	@Override

	protected List<String> toList(final Field list) {

		FieldList fl = (FieldList) list;

		List<String> values = Lists.newArrayList(fl.stringList());

		Collections.sort(values);

		return values;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.List;

import java.util.Map;

import com.google.common.collect.Lists;

import eu.dnetlib.pace.model.Person;

public class PersonHash extends AbstractClusteringFunction {

	private boolean DEFAULT_AGGRESSIVE = false;

	public PersonHash(final Map<String, Integer> params) {

		super(params);

	}

	@Override

	protected Collection<String> doApply(final String s) {

		final List<String> res = Lists.newArrayList();

		final boolean aggressive = (Boolean) (getParams().containsKey("aggressive") ? getParams().get("aggressive") : DEFAULT_AGGRESSIVE);

		res.add(new Person(s, aggressive).hash());

		return res;

	}

}

package eu.dnetlib.pace.distance;

import org.apache.commons.lang.StringUtils;

import com.wcohen.ss.AbstractStringDistance;

import eu.dnetlib.pace.config.Type;

import eu.dnetlib.pace.model.Field;

/**

 * The Class SubStringLevenstein.

 */

public class SubStringLevenstein extends SecondStringDistanceAlgo {

	/** The limit. */

	protected int limit;

	/**

	 * Instantiates a new sub string levenstein.

	 * 

	 * @param w

	 *            the w

	 */

	public SubStringLevenstein(final double w) {

		super(w, new com.wcohen.ss.Levenstein());

	}

	/**

	 * Instantiates a new sub string levenstein.

	 * 

	 * @param w

	 *            the w

	 * @param limit

	 *            the limit

	 */

	public SubStringLevenstein(final double w, final int limit) {

		super(w, new com.wcohen.ss.Levenstein());

		this.limit = limit;

	}

	/**

	 * Instantiates a new sub string levenstein.

	 * 

	 * @param w

	 *            the w

	 * @param limit

	 *            the limit

	 * @param ssalgo

	 *            the ssalgo

	 */

	protected SubStringLevenstein(final double w, final int limit, final AbstractStringDistance ssalgo) {

		super(w, ssalgo);

		this.limit = limit;

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.distance.SecondStringDistanceAlgo#distance(eu.dnetlib.pace.model.Field, eu.dnetlib.pace.model.Field)

	 */

	@Override

	public double distance(final Field a, final Field b) {

		if (a.getType().equals(Type.String) && b.getType().equals(Type.String))

			return distance(StringUtils.left(a.stringValue(), limit), StringUtils.left(b.stringValue(), limit));

		throw new IllegalArgumentException("invalid types\n- A: " + a.toString() + "\n- B: " + b.toString());

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.distance.DistanceAlgo#getWeight()

	 */

	@Override

	public double getWeight() {

		return super.weight;

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.distance.SecondStringDistanceAlgo#normalize(double)

	 */

	@Override

	protected double normalize(final double d) {

		return 1 / Math.pow(Math.abs(d) + 1, 0.1);

	}

}

package eu.dnetlib.pace.distance;

import com.wcohen.ss.AbstractStringDistance;

/**

 * The Class SortedJaroWinkler.

 */

public class SortedLevel2JaroWinkler extends SortedSecondStringDistanceAlgo {

	/**

	 * Instantiates a new sorted jaro winkler.

	 * 

	 * @param weight

	 *            the weight

	 */

	public SortedLevel2JaroWinkler(final double weight) {

		super(weight, new com.wcohen.ss.Level2JaroWinkler());

	}

	/**

	 * Instantiates a new sorted jaro winkler.

	 * 

	 * @param weight

	 *            the weight

	 * @param ssalgo

	 *            the ssalgo

	 */

	protected SortedLevel2JaroWinkler(final double weight, final AbstractStringDistance ssalgo) {

		super(weight, ssalgo);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.distance.DistanceAlgo#getWeight()

	 */

	@Override

	public double getWeight() {

		return super.weight;

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.distance.SecondStringDistanceAlgo#normalize(double)

	 */

	@Override

	protected double normalize(final double d) {

		return d;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.List;

import java.util.Map;

import com.google.common.base.Predicate;

import eu.dnetlib.pace.model.Field;

public class FieldFilter implements Predicate<Field> {

	private Map<String, List<String>> blacklists;

	private String filedName;

	public FieldFilter(final String fieldName, final Map<String, List<String>> blacklists) {

		this.filedName = fieldName;

		this.blacklists = blacklists;

	}

	@Override

	public boolean apply(final Field f) {

		return !regexMatches(filedName, f.stringValue(), blacklists);

	}

	/**

	 * Tries to match the fields in the regex blacklist.

	 *

	 * @param fieldName

	 * @param value

	 * @return true if the field matches, false otherwise

	 */

	protected boolean regexMatches(final String fieldName, final String value, final Map<String, List<String>> blacklists) {

		if (blacklists.containsKey(fieldName)) {

			final Iterable<String> regexes = blacklists.get(fieldName);

			for (final String regex : regexes) {

				if (value.matches(regex)) return true;

			}

		}

		return false;

	}

}

package eu.dnetlib.pace.distance;

public class YearLevenstein extends SubStringLevenstein {

	public YearLevenstein(double w) {

		super(w);

	}

	public YearLevenstein(double w, int limit) {

		super(w, limit);	

	}

	@Override

	public double distance(String a, String b) {

		boolean check = checkLength(a) && checkLength(b);

		if (check) {

			if (a.equals(b)) {

				return 1.0;

			} else {

				return 0.5;

			}

		} else {

			return 1.0;

		}

	}

	protected boolean checkLength(String s) {

		return getNumbers(s).length() == limit;

	}

	@Override

	public double getWeight() {

		return super.weight;

	}

}

package eu.dnetlib.pace.distance;

import eu.dnetlib.pace.model.Field;

/**

 * Not all fields of a document need to partecipate in the distance measure. We model those fields as having a

 * NullDistanceAlgo.

 */

public class NullDistanceAlgo implements DistanceAlgo {

	@Override

	public double distance(Field a, Field b) {

		return 0.0;

	}

	@Override

	public double getWeight() {

		return 0.0;

	}

}

package eu.dnetlib.pace.distance;

import com.wcohen.ss.AbstractStringDistance;

public class Level2Levenstein extends SecondStringDistanceAlgo {

	public Level2Levenstein(double w) {

		super(w, new com.wcohen.ss.Level2Levenstein());

	}

	protected Level2Levenstein(double w, AbstractStringDistance ssalgo) {

		super(w, ssalgo);

	}

	@Override

	public double getWeight() {

		return super.weight;

	}

	@Override

	protected double normalize(double d) {

		return 1 / Math.pow(Math.abs(d) + 1, 0.1);

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet-hadoop-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-pace-core</artifactId>

	<packaging>jar</packaging>

	<version>2.1.2</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-pace-core/tags/dnet-pace-core-2.1.2</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>edu.cmu</groupId>

			<artifactId>secondstring</artifactId>

			<version>1.0.0</version>

		</dependency>

		<dependency>

			<groupId>com.google.guava</groupId>

			<artifactId>guava</artifactId>

			<version>${google.guava.version}</version>

		</dependency>

		<dependency>

			<groupId>com.google.code.gson</groupId>

			<artifactId>gson</artifactId>

			<version>${google.gson.version}</version>

		</dependency>

		<dependency>

			<groupId>commons-lang</groupId>

			<artifactId>commons-lang</artifactId>

			<version>${commons.lang.version}</version>

		</dependency>

		<dependency>

			<groupId>commons-io</groupId>

			<artifactId>commons-io</artifactId>

			<version>${commons.io.version}</version>

		</dependency>

		<dependency>

			<groupId>commons-collections</groupId>

			<artifactId>commons-collections</artifactId>

			<version>${commons.collections.version}</version>

		</dependency>

		<dependency>

			<groupId>org.antlr</groupId>

			<artifactId>stringtemplate</artifactId>

			<version>3.2</version>

		</dependency>	

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

	</dependencies>

</project>

package eu.dnetlib.pace.clustering;

public enum Clustering {

	acronyms, ngrams, ngrampairs, suffixprefix, spacetrimmingfieldvalue, immutablefieldvalue, personhash

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.List;

import java.util.Map;

import java.util.Map.Entry;

import java.util.Set;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import eu.dnetlib.pace.config.Config;

import eu.dnetlib.pace.model.Document;

import eu.dnetlib.pace.model.FieldListImpl;

import eu.dnetlib.pace.model.MapDocument;

public class BlacklistAwareClusteringCombiner extends ClusteringCombiner {

	public static Collection<String> filterAndCombine(final MapDocument a, final Config conf, final Map<String, List<String>> blacklists) {

		final Document filtered = new BlacklistAwareClusteringCombiner().filter(a, blacklists);

		return combine(filtered, conf);

	}

	private MapDocument filter(final MapDocument a, final Map<String, List<String>> blacklists) {

		final Map<String, FieldListImpl> filtered = Maps.newHashMap(a.getFieldMap());

		if (blacklists != null) {

			for (final Entry<String, FieldListImpl> e : filtered.entrySet()) {

				final FieldListImpl fl = new FieldListImpl();

				fl.addAll(Lists.newArrayList(Iterables.filter(e.getValue(), new FieldFilter(e.getKey(), blacklists))));

				filtered.put(e.getKey(), fl);

			}

		}

		return new MapDocument(a.getIdentifier(), filtered);

	}

	/**

	 * Tries to match the fields in the regex blacklist.

	 *

	 * @param fieldName

	 * @param value

	 * @return true if the field matches, false otherwise

	 */

	protected boolean regexMatches(final String fieldName, final String value, final Map<String, Set<String>> blacklists) {

		if (blacklists.containsKey(fieldName)) {

			for (final String regex : blacklists.get(fieldName)) {

				if (value.matches(regex)) return true;

			}

		}

		return false;

	}

}

package eu.dnetlib.pace.model;

import java.util.List;

import com.google.common.base.Splitter;

import com.google.common.collect.Lists;

import com.google.gson.Gson;

import eu.dnetlib.pace.config.Algo;

import eu.dnetlib.pace.config.Type;

import eu.dnetlib.pace.distance.AlwaysMatch;

import eu.dnetlib.pace.distance.DistanceAlgo;

import eu.dnetlib.pace.distance.ExactMatch;

import eu.dnetlib.pace.distance.JaroWinkler;

import eu.dnetlib.pace.distance.JaroWinklerTitle;

import eu.dnetlib.pace.distance.Level2JaroWinkler;

import eu.dnetlib.pace.distance.Level2JaroWinklerTitle;

import eu.dnetlib.pace.distance.Level2Levenstein;

import eu.dnetlib.pace.distance.Levenstein;

import eu.dnetlib.pace.distance.LevensteinTitle;

import eu.dnetlib.pace.distance.NullDistanceAlgo;

import eu.dnetlib.pace.distance.SortedJaroWinkler;

import eu.dnetlib.pace.distance.SortedLevel2JaroWinkler;

import eu.dnetlib.pace.distance.SubStringLevenstein;

import eu.dnetlib.pace.distance.YearLevenstein;

/**

 * The schema is composed by field definitions (FieldDef). Each field has a type, a name, and an associated distance algorithm.

 */

public class FieldDef {

	public final static String PATH_SEPARATOR = "/";

	private Algo algo;

	private String name;

	private String path;

	private boolean ignoreMissing;

	private Type type;

	private boolean overrideMatch;

	private double weight;

	private int limit = -1;

	public FieldDef() {}

	// def apply(s: String): Field[A]

	public Field apply(final Type type, final String s) {

		switch (type) {

		case Int:

			return new FieldValueImpl(type, name, Integer.parseInt(s));

		case String:

			return new FieldValueImpl(type, name, s);

		case List:

			return new FieldListImpl(name);

		default:

			throw new IllegalArgumentException("Casting not implemented for type " + type);

		}

	}

	public String getName() {

		return name;

	}

	public String getPath() {

		return path;

	}

	public List<String> getPathList() {

		return Lists.newArrayList(Splitter.on(PATH_SEPARATOR).split(getPath()));

	}

	public DistanceAlgo getDistanceAlgo() {

		switch (getAlgo()) {

		case JaroWinkler:

			return new JaroWinkler(getWeight());

		case JaroWinklerTitle:

			return new JaroWinklerTitle(getWeight());

		case Level2JaroWinkler:

			return new Level2JaroWinkler(getWeight());

		case Level2JaroWinklerTitle:

			return new Level2JaroWinklerTitle(getWeight());

		case Level2Levenstein:

			return new Level2Levenstein(getWeight());

		case Levenstein:

			return new Levenstein(getWeight());

		case LevensteinTitle:

			return new LevensteinTitle(getWeight());

		case SubStringLevenstein:

			return new SubStringLevenstein(getWeight(), getLimit());

		case YearLevenstein:

			return new YearLevenstein(getWeight(), getLimit());

		case SortedJaroWinkler:

			return new SortedJaroWinkler(getWeight());

		case SortedLevel2JaroWinkler:

			return new SortedLevel2JaroWinkler(getWeight());

		case ExactMatch:

			return new ExactMatch(getWeight());

		case AlwaysMatch:

			return new AlwaysMatch(getWeight());

		case Null:

			return new NullDistanceAlgo();

		default:

			return new NullDistanceAlgo();

		}

	}

	public boolean isIgnoreMissing() {

		return ignoreMissing;

	}

	public Type getType() {

		return type;

	}

	public void setType(final Type type) {

		this.type = type;

	}

	public boolean isOverrideMatch() {

		return overrideMatch;

	}

	public void setOverrideMatch(final boolean overrideMatch) {

		this.overrideMatch = overrideMatch;

	}

	@Override

	public String toString() {

		return new Gson().toJson(this);

	}

	public double getWeight() {

		return weight;

	}

	public void setWeight(final double weight) {

		this.weight = weight;

	}

	public Algo getAlgo() {

		return algo;

	}

	public void setAlgo(final Algo algo) {

		this.algo = algo;

	}

	public int getLimit() {

		return limit;

	}

	public void setLimit(final int limit) {

		this.limit = limit;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.List;

import java.util.Map;

import org.apache.commons.lang.RandomStringUtils;

import org.apache.commons.lang.StringUtils;

import com.google.common.collect.Lists;

public class SpaceTrimmingFieldValue extends AbstractClusteringFunction {

	public SpaceTrimmingFieldValue(final Map<String, Integer> params) {

		super(params);

	}

	@Override

	protected Collection<String> doApply(final String s) {

		final List<String> res = Lists.newArrayList();

		res.add(StringUtils.isBlank(s) ? RandomStringUtils.random(getParams().get("randomLength")) : s.toLowerCase().replaceAll("\\s+", ""));

		return res;

	}

}

package eu.dnetlib.pace.model;

import java.util.List;

import java.util.Map;

import com.google.gson.Gson;

import eu.dnetlib.pace.clustering.Acronyms;

import eu.dnetlib.pace.clustering.Clustering;

import eu.dnetlib.pace.clustering.ClusteringFunction;

import eu.dnetlib.pace.clustering.ImmutableFieldValue;

import eu.dnetlib.pace.clustering.NgramPairs;

import eu.dnetlib.pace.clustering.Ngrams;

import eu.dnetlib.pace.clustering.PersonHash;

import eu.dnetlib.pace.clustering.RandomClusteringFunction;

import eu.dnetlib.pace.clustering.SpaceTrimmingFieldValue;

import eu.dnetlib.pace.clustering.SuffixPrefix;

public class ClusteringDef {

	private Clustering name;

	private List<String> fields;

	private Map<String, Integer> params;

	public ClusteringDef() {}

	public Clustering getName() {

		return name;

	}

	public void setName(final Clustering name) {

		this.name = name;

	}

	public ClusteringFunction getClusteringFunction() {

		switch (getName()) {

		case acronyms:

			return new Acronyms(getParams());

		case ngrams:

			return new Ngrams(getParams());

		case ngrampairs:

			return new NgramPairs(getParams());

		case suffixprefix:

			return new SuffixPrefix(getParams());

		case spacetrimmingfieldvalue:

			return new SpaceTrimmingFieldValue(getParams());

		case immutablefieldvalue:

			return new ImmutableFieldValue(getParams());

		case personhash:

			return new PersonHash(getParams());

		default:

			return new RandomClusteringFunction(getParams());

		}

	}

	public List<String> getFields() {

		return fields;

	}

	public void setFields(final List<String> fields) {

		this.fields = fields;

	}

	public Map<String, Integer> getParams() {

		return params;

	}

	public void setParams(final Map<String, Integer> params) {

		this.params = params;

	}

	@Override

	public String toString() {

		return new Gson().toJson(this);

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.LinkedHashSet;

import java.util.Map;

import java.util.StringTokenizer;

public class Ngrams extends AbstractClusteringFunction {

	public Ngrams(Map<String, Integer> params) {

		super(params);

	}

	@Override

	protected Collection<String> doApply(String s) {

		return getNgrams(s, param("ngramLen"), param("max"), param("maxPerToken"), param("minNgramLen"));

	}

	protected Collection<String> getNgrams(String s, int ngramLen, int max, int maxPerToken, int minNgramLen) {

		final Collection<String> ngrams = new LinkedHashSet<String>();

		final StringTokenizer st = new StringTokenizer(s);

		while (st.hasMoreTokens()) {

			final String token = st.nextToken();

			if (!token.isEmpty()) {

				for (int i = 0; i < maxPerToken && ngramLen + i <= token.length(); i++) {

					String ngram = (token + "    ").substring(i, ngramLen + i).trim();

					if (ngrams.size() >= max) {

						return ngrams;

					}

					if (ngram.length() >= minNgramLen) {

						ngrams.add(ngram);

					}

				}

			}

		}

		//System.out.println(ngrams + " n: " + ngrams.size());

		return ngrams;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.Map;

import java.util.Set;

import java.util.StringTokenizer;

import com.google.common.collect.Sets;

public class Acronyms extends AbstractClusteringFunction {

	public Acronyms(Map<String, Integer> params) {

		super(params);

	}

	@Override

	protected Collection<String> doApply(String s) {

		return extractAcronyms(s, param("max"), param("minLen"), param("maxLen"));

	}

	private Set<String> extractAcronyms(final String s, int maxAcronyms, int minLen, int maxLen) {

		final Set<String> acronyms = Sets.newLinkedHashSet();

		for (int i = 0; i < maxAcronyms; i++) {

			final StringTokenizer st = new StringTokenizer(s);

			final StringBuilder sb = new StringBuilder();

			while (st.hasMoreTokens()) {

				final String token = st.nextToken();

				if (sb.length() > maxLen) {

					break;

				}

				if (token.length() > 1 && i < token.length()) {

					sb.append(token.charAt(i));

				}

			}

			String acronym = sb.toString();

			if (acronym.length() > minLen) {

				acronyms.add(acronym);

			}

		}

		return acronyms;

	}

}

package eu.dnetlib.pace.model;

import java.util.List;

import com.google.gson.Gson;

import eu.dnetlib.pace.condition.AlwaysTrueCondition;

import eu.dnetlib.pace.condition.ConditionAlgo;

import eu.dnetlib.pace.condition.DoiExactMatch;

import eu.dnetlib.pace.condition.ExactMatch;

import eu.dnetlib.pace.condition.SizeMatch;

import eu.dnetlib.pace.condition.TitleVersionMatch;

import eu.dnetlib.pace.condition.YearMatch;

import eu.dnetlib.pace.config.Cond;

public class CondDef {

	private Cond name;

	private List<String> fields;

	public CondDef() {}

	public ConditionAlgo getConditionAlgo(final List<FieldDef> fields) {

		switch (getName()) {

		case yearMatch:

			return new YearMatch(fields);

		case titleVersionMatch:

			return new TitleVersionMatch(fields);

		case sizeMatch:

			return new SizeMatch(fields);

		case exactMatch:

			return new ExactMatch(fields);

		case doiExactMatch:

			return new DoiExactMatch(fields);

		default:

			return new AlwaysTrueCondition(fields);

		}

	}

	public Cond getName() {

		return name;

	}

	public void setName(final Cond name) {

		this.name = name;

	}

	public List<String> getFields() {

		return fields;

	}

	public void setFields(final List<String> fields) {

		this.fields = fields;

	}

	@Override

	public String toString() {

		return new Gson().toJson(this);

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Set;

import org.apache.commons.lang.StringUtils;

import eu.dnetlib.pace.common.AbstractPaceFunctions;

public class NGramUtils extends AbstractPaceFunctions {

	private static final int SIZE = 100;

	private static Set<String> stopwords = AbstractPaceFunctions.loadFromClasspath("/eu/dnetlib/pace/config/stopwords_en.txt");

	public static String cleanupForOrdering(String s) {

		NGramUtils utils = new NGramUtils();

		return (utils.filterStopWords(utils.normalize(s), stopwords) +  StringUtils.repeat(" ", SIZE)).substring(0, SIZE).replaceAll(" ", "");

	}

}

package eu.dnetlib.pace.model;

import java.util.Collection;

import java.util.Iterator;

import java.util.List;

import java.util.ListIterator;

import com.google.common.base.Function;

import com.google.common.base.Joiner;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import eu.dnetlib.pace.config.Type;

/**

 * The Class FieldListImpl.

 */

public class FieldListImpl extends AbstractField implements FieldList {

	/** The fields. */

	private List<Field> fields;

	/**

	 * Instantiates a new field list impl.

	 */

	public FieldListImpl() {

		fields = Lists.newArrayList();

	}

	/**

	 * Instantiates a new field list impl.

	 * 

	 * @param name

	 *            the name

	 */

	public FieldListImpl(final String name) {

		super(Type.List, name);

		fields = Lists.newArrayList();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#add(java.lang.Object)

	 */

	@Override

	public boolean add(final Field f) {

		return fields.add(f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#add(int, java.lang.Object)

	 */

	@Override

	public void add(final int i, final Field f) {

		fields.add(i, f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#addAll(java.util.Collection)

	 */

	@Override

	public boolean addAll(final Collection<? extends Field> f) {

		return fields.addAll(f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#addAll(int, java.util.Collection)

	 */

	@Override

	public boolean addAll(final int i, final Collection<? extends Field> f) {

		return fields.addAll(i, f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#clear()

	 */

	@Override

	public void clear() {

		fields.clear();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#contains(java.lang.Object)

	 */

	@Override

	public boolean contains(final Object o) {

		return fields.contains(o);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#containsAll(java.util.Collection)

	 */

	@Override

	public boolean containsAll(final Collection<?> f) {

		return fields.containsAll(f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#get(int)

	 */

	@Override

	public Field get(final int i) {

		return fields.get(i);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#indexOf(java.lang.Object)

	 */

	@Override

	public int indexOf(final Object o) {

		return fields.indexOf(o);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.model.Field#isEmpty()

	 */

	@Override

	public boolean isEmpty() {

		return fields.isEmpty();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.lang.Iterable#iterator()

	 */

	@Override

	public Iterator<Field> iterator() {

		return fields.iterator();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#lastIndexOf(java.lang.Object)

	 */

	@Override

	public int lastIndexOf(final Object o) {

		return fields.lastIndexOf(o);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#listIterator()

	 */

	@Override

	public ListIterator<Field> listIterator() {

		return fields.listIterator();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#listIterator(int)

	 */

	@Override

	public ListIterator<Field> listIterator(final int i) {

		return fields.listIterator(i);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#remove(java.lang.Object)