D-Net

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-pace-core/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "dnet-pace-core"}

package eu.dnetlib.pace.clustering;

import java.util.Map;

import org.junit.Before;

import org.junit.Test;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import eu.dnetlib.pace.AbstractPaceTest;

public class ClusteringFunctionTest extends AbstractPaceTest {

	private Map<String, Integer> params;

	@Before

	public void setUp() throws Exception {

		params = Maps.newHashMap();

	}

	@Test

	public void testNgram() {

		params.put("ngramLen", 2);

		params.put("max", 8);

		params.put("maxPerToken", 1);

		params.put("minNgramLen", 3);

		final ClusteringFunction ngram = new Ngrams(params);

		final String s = "Search for the Standard Model Higgs Boson";

		System.out.println(s);

		System.out.println(ngram.apply(Lists.newArrayList(title(s))));

	}

	@Test

	public void testNgramPairs() {

		params.put("ngramLen", 2);

		params.put("max", 4);

		final ClusteringFunction np = new NgramPairs(params);

		final String s = "Search for the Standard Model Higgs Boson";

		System.out.println(s);

		System.out.println(np.apply(Lists.newArrayList(title(s))));

	}

	@Test

	public void testAcronym() {

		params.put("max", 4);

		params.put("minLen", 1);

		params.put("maxLen", 3);

		final ClusteringFunction acro = new Acronyms(params);

		final String s = "Search for the Standard Model Higgs Boson";

		System.out.println(s);

		System.out.println(acro.apply(Lists.newArrayList(title(s))));

	}

	@Test

	public void testSuffixPrefix() {

		params.put("len", 2);

		params.put("max", 3);

		final ClusteringFunction sp = new SuffixPrefix(params);

		final String s = "Search for the Standard Model Higgs Boson";

		System.out.println(s);

		System.out.println(sp.apply(Lists.newArrayList(title(s))));

	}

	@Test

	public void testFieldValue() {

		final ClusteringFunction sp = new SpaceTrimmingFieldValue(params);

		final String s = "Search for the Standard Model Higgs Boson";

		System.out.println(s);

		System.out.println(sp.apply(Lists.newArrayList(title(s))));

	}

}

package eu.dnetlib.pace.model;

import static org.junit.Assert.assertFalse;

import static org.junit.Assert.assertTrue;

import org.junit.Test;

public class PersonComparatorUtilsSimilarityTest {

	@Test

	public void testSimilarity_0() {

		assertTrue(PersonComparatorUtils.areSimilar("Artini Michele", "Michele Artini"));

	}

	@Test

	public void testSimilarity_1() {

		assertTrue(PersonComparatorUtils.areSimilar("ARTINI Michele", "Artini, Michele"));

	}

	@Test

	public void testSimilarity_2() {

		assertTrue(PersonComparatorUtils.areSimilar("Artini, M.", "Artini Michele"));

	}

	@Test

	public void testSimilarity_3() {

		assertTrue(PersonComparatorUtils.areSimilar("Artini, M.G.", "Artini, Michele"));

	}

	@Test

	public void testSimilarity_4() {

		assertTrue(PersonComparatorUtils.areSimilar("Artini, M.", "Artini, M.G."));

	}

	@Test

	public void testSimilarity_5() {

		assertTrue(PersonComparatorUtils.areSimilar("Artini, M. (sig.)", "Artini, Michele"));

	}

	@Test

	public void testSimilarity_6() {

		assertFalse(PersonComparatorUtils.areSimilar("Artini, M.", "Artini, G."));

	}

	@Test

	public void testSimilarity_7() {

		assertFalse(PersonComparatorUtils.areSimilar("Artini, M.G.", "Artini, M.A."));

	}

	@Test

	public void testSimilarity_8() {

		assertFalse(PersonComparatorUtils.areSimilar("Artini, M.", "Artini, Giuseppe"));

	}

	@Test

	public void testSimilarity_9() {

		assertFalse(PersonComparatorUtils.areSimilar("Manghi, Paolo", "Artini, Michele"));

	}

	@Test

	public void testSimilarity_10() {

		assertTrue(PersonComparatorUtils.areSimilar("Artini, Michele", "Artini, Michele Giovanni"));

	}

	@Test

	public void testSimilarity_11() {

		assertFalse(PersonComparatorUtils.areSimilar("Artini, M.A.G.", "Artini, M.B.G."));

	}

	@Test

	public void testSimilarity_12() {

		assertFalse(PersonComparatorUtils.areSimilar("Artini Manghi, M.", "Artini, Michele"));

	}

	@Test

	public void testSimilarity_13() {

		assertTrue(PersonComparatorUtils.areSimilar("Artini Manghi, M.", "Artini Manghi Michele"));

	}

	@Test

	public void testSimilarity_14() {

		assertFalse(PersonComparatorUtils.areSimilar("Artini, Michele", "Michele, Artini"));

	}

	@Test

	public void testSimilarity_15() {

		assertTrue(PersonComparatorUtils.areSimilar("Artini, M.", "Michele ARTINI"));

	}

}

package eu.dnetlib.pace.model;

import static org.junit.Assert.assertEquals;

import java.text.Normalizer;

import java.util.Queue;

import org.junit.Test;

import com.google.common.collect.Lists;

public class PersonTest {

	@Test

	public void test_1() {

		check("Atzori, Claudio", "Atzori, Claudio");

	}

	@Test

	public void test_2() {

		check("Atzori, Claudio A.", "Atzori, Claudio A.");

	}

	@Test

	public void test_3() {

		check("Claudio ATZORI", "Atzori, Claudio");

	}

	@Test

	public void test_4() {

		check("ATZORI, Claudio", "Atzori, Claudio");

	}

	@Test

	public void test_5() {

		check("Claudio Atzori", "Claudio Atzori");

	}

	@Test

	public void test_6() {

		check(" Manghi ,  Paolo", "Manghi, Paolo");

	}

	@Test

	public void test_7() {

		check("ATZORI, CLAUDIO", "Atzori, Claudio");

	}

	@Test

	public void test_8() {

		check("ATZORI, CLAUDIO A", "Atzori, Claudio A.");

	}

	@Test

	public void test_9() {

		check("Bølviken, B.", "Bølviken, B.");

	}

	@Test

	public void test_10() {

		check("Bñlviken, B.", "B" + Normalizer.normalize("ñ", Normalizer.Form.NFD) + "lviken, B.");

	}

	@Test

	public void test_11() {

		check("aáeéiíoóöőuúüű AÁEÉIÍOÓÖŐUÚÜŰ ø", "Aaeeiioooouuuu, Aaeeiioooouuuu Ø.", true);

	}

	@Test

	public void test_12() {

		check("aáeéiíoóöőuúüű AÁEÉIÍOÓÖŐUÚÜŰz ø", Normalizer.normalize("aáeéiíoóöőuúüű AÁEÉIÍOÓÖŐUÚÜŰz ø", Normalizer.Form.NFD), false);

	}

	@Test

	public void test_13() {

		check("Tkačíková, Daniela", Normalizer.normalize("Tkačíková, Daniela", Normalizer.Form.NFD), false);

	}

	@Test

	public void test_hashes() {

		checkHash(" Claudio  ATZORI ", "ATZORI Claudio", "Atzori , Claudio", "ATZORI, Claudio");

	}

	private void checkHash(String... ss) {

		Queue<String> q = Lists.newLinkedList(Lists.newArrayList(ss));

		String h1 = new Person(q.remove(), false).hash();

		while (!q.isEmpty()) {

			assertEquals(h1, new Person(q.remove(), false).hash());

		}

	}

	private void check(String s, String expectedFullName) {

		check(s, expectedFullName, false);

	}

	private void check(String s, String expectedFullName, boolean aggressive) {

		Person p = new Person(s, aggressive);

		System.out.println("original:   " + p.getOriginal());

		System.out.println("accurate:   " + p.isAccurate());

		System.out.println("normalised: '" + p.getNormalisedFullname() + "'");

		if (p.isAccurate()) {

			System.out.println("name:     " + p.getNormalisedFirstName());

			System.out.println("surname:  " + p.getNormalisedSurname());

		}

		System.out.println("hash: " + p.hash());

		System.out.println("");

		assertEquals(expectedFullName, p.getNormalisedFullname());

	}

}

package eu.dnetlib.pace.model;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertTrue;

import java.util.Set;

import org.junit.Test;

public class PersonComparatorUtilsNGramsTest {

	@Test

	public void testNormaizePerson_1() {

		verifyGetNgramsForPerson("Artini Michele", 2, "a_michele", "m_artini");

	}

	@Test

	public void testNormaizePerson_2() {

		verifyGetNgramsForPerson("Michele Artini", 2, "a_michele", "m_artini");

	}

	@Test

	public void testNormaizePerson_3() {

		verifyGetNgramsForPerson("Michele ARTINI", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_4() {

		verifyGetNgramsForPerson("ARTINI Michele", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_5() {

		verifyGetNgramsForPerson("Michele G. Artini", 2, "m_artini", "g_artini");

	}

	@Test

	public void testNormaizePerson_6() {

		verifyGetNgramsForPerson(" Artini, Michele ", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_7() {

		verifyGetNgramsForPerson("Artini, Michele (sig.)", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_8() {

		verifyGetNgramsForPerson("Artini Michele [sig.] ", 2, "a_michele", "m_artini");

	}

	@Test

	public void testNormaizePerson_9() {

		verifyGetNgramsForPerson("Artini, M", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_10() {

		verifyGetNgramsForPerson("Artini, M.", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_11() {

		verifyGetNgramsForPerson("Artini, M. (sig.)", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_12() {

		verifyGetNgramsForPerson("Artini, M[sig.] ", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_13() {

		verifyGetNgramsForPerson("Artini-SIG, Michele ", 1, "m_artini-sig");

	}

	@Test

	public void testNormaizePerson_14() {

		verifyGetNgramsForPerson("Artini - SIG, Michele ", 1, "m_artini-sig");

	}

	@Test

	public void testNormaizePerson_15() {

		verifyGetNgramsForPerson("Artini {sig.}, M", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_16() {

		verifyGetNgramsForPerson("Artini, M., sig.", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_17() {

		verifyGetNgramsForPerson("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA, BBBBBBBBBBBBBBBBBBBBBBBBBBBBB CCCCCCCCCCCCCCCCCCCC", 0);

	}

	@Test

	public void testNormaizePerson_18() {

		verifyGetNgramsForPerson("Dell'amico, Andrea", 1, "a_amico");

	}

	@Test

	public void testNormaizePerson_19() {

		verifyGetNgramsForPerson("Smith, Paul van der", 1, "p_smith");

	}

	@Test

	public void testNormaizePerson_20() {

		verifyGetNgramsForPerson("AAAAAAA, BBBB, CCCC, DDDD, EEEE", 1, "b_aaaaaaa");

	}

	@Test

	public void testNormaizePerson_21() {

		verifyGetNgramsForPerson("Kompetenzzentrum Informelle Bildung (KIB),", 6);

	}

	private void verifyGetNgramsForPerson(String name, int expectedSize, String... expectedTokens) {

		Set<String> list = PersonComparatorUtils.getNgramsForPerson(name);

		System.out.println(list);

		assertEquals(expectedSize, list.size());

		for (String s : expectedTokens) {

			assertTrue(list.contains(s));

		}

	}

}

package eu.dnetlib.pace.config;

import static org.junit.Assert.assertNotNull;

import java.io.IOException;

import org.junit.Test;

import eu.dnetlib.pace.AbstractPaceTest;

public class ConfigTest extends AbstractPaceTest {

	@Test

	public void test() throws IOException {

		final DedupConfig cfg = DedupConfig.load(readFromClasspath("result.pace.conf.json"));

		assertNotNull(cfg);

		System.out.println(cfg);

	}

}

package eu.dnetlib.pace.distance;

import org.junit.Before;

import org.junit.Test;

import eu.dnetlib.pace.common.AbstractPaceFunctions;

public class DistanceAlgoTest extends AbstractPaceFunctions {

	private final static String TEST_STRING = "Toshiba NB550D: è un netbook su piattaforma AMD Fusion⁽¹²⁾.";

	@Before

	public void setup() {

		System.out.println("****************************************************************");

		System.out.println("Test String    : " + TEST_STRING);

	}

	@Test

	public void testGetNumbers() {

		System.out.println("Numbers        : " + getNumbers(TEST_STRING));

	}

	@Test

	public void testRemoveSymbols() {

		System.out.println("Without symbols: " + removeSymbols(TEST_STRING));

	}

	@Test

	public void testFixAliases() {

		System.out.println("Fixed aliases  : " + fixAliases(TEST_STRING));

	}

	@Test

	public void testCleanup() {

		System.out.println("cleaned up     : " + cleanup(TEST_STRING));

	}

}

package eu.dnetlib.pace;

import java.io.IOException;

import java.io.StringWriter;

import org.apache.commons.io.IOUtils;

import eu.dnetlib.pace.config.Type;

import eu.dnetlib.pace.model.Field;

import eu.dnetlib.pace.model.FieldValueImpl;

public abstract class AbstractPaceTest {

	protected String readFromClasspath(final String filename) {

		final StringWriter sw = new StringWriter();

		try {

			IOUtils.copy(getClass().getResourceAsStream(filename), sw);

			return sw.toString();

		} catch (final IOException e) {

			throw new RuntimeException("cannot load resource from classpath: " + filename);

		}

	}

	protected Field title(final String s) {

		return new FieldValueImpl(Type.String, "title", s);

	}

}

{ 

	"wf" : { 

        "threshold" : "0.99", 

        "run" : "001", 

        "entityType" : "result", 

        "orderField" : "title", 

        "queueMaxSize" : "2000",

        "groupMaxSize" : "10",

        "slidingWindowSize" : "200",

        "rootBuilder" : [ "result" ],

        "includeChildren" : "true" 

    },

	"pace" : {		

		"clustering" : [

			{ "name" : "acronyms", "fields" : [ "title" ], "params" : { "max" : "1", "minLen" : "2", "maxLen" : "4"} },

			{ "name" : "ngrampairs", "fields" : [ "title" ], "params" : { "max" : "1", "ngramLen" : "3"} },

			{ "name" : "suffixprefix", "fields" : [ "title" ], "params" : { "max" : "1", "len" : "3" } } 

		],		

		"strictConditions" : [

  			{ "name" : "exactMatch", "fields" : [ "pid" ] }

  		], 

  		"conditions" : [ 

  			{ "name" : "yearMatch", "fields" : [ "dateofacceptance" ] },

  			{ "name" : "titleVersionMatch", "fields" : [ "title" ] },

  			{ "name" : "sizeMatch", "fields" : [ "authors" ] } 

  		],		

		"model" : [

			{ "name" : "pid", "algo" : "Null", "type" : "String", "weight" : "0.0", "ignoreMissing" : "true", "path" : "pid/value", "overrideMatch" : "true" }, 	

			{ "name" : "title", "algo" : "JaroWinkler", "type" : "String", "weight" : "1.0", "ignoreMissing" : "false", "path" : "result/metadata/title/value" },

			{ "name" : "dateofacceptance", "algo" : "Null", "type" : "String", "weight" : "0.0", "ignoreMissing" : "true", "path" : "result/metadata/dateofacceptance/value" } ,

			{ "name" : "authors", "algo" : "Null", "type" : "List", "weight" : "0.0", "ignoreMissing" : "true", "path" : "result/author/metadata/fullname/value" } 		

		],

		"blacklists" : {

			"title" : [

				"^(Corpus Oral Dialectal \\(COD\\)\\.).*$",

				"^(Kiri Karl Morgensternile).*$",

				"^(\\[Eksliibris Aleksandr).*\\]$",

				"^(\\[Eksliibris Aleksandr).*$",

				"^(Eksliibris Aleksandr).*$",

				"^(Kiri A\\. de Vignolles).*$",

				"^(2 kirja Karl Morgensternile).*$",

				"^(Pirita kloostri idaosa arheoloogilised).*$",

				"^(Kiri tundmatule).*$",

				"^(Kiri Jenaer Allgemeine Literaturzeitung toimetusele).*$",

				"^(Eksliibris Nikolai Birukovile).*$",

				"^(Eksliibris Nikolai Issakovile).*$",

				"^(WHP Cruise Summary Information of section).*$",

				"^(Measurement of the top quark\\-pair production cross section with ATLAS in pp collisions at).*$",

				"^(Measurement of the spin\\-dependent structure function).*"

			] } 		

	}

}

^(Corpus Oral Dialectal \(COD\)\.).*$

^(Kiri Karl Morgensternile).*$

^(\[Eksliibris Aleksandr).*\]$

^(Kiri A\. de Vignolles).*$

^(2 kirja Karl Morgensternile).*$

^(Pirita kloostri idaosa arheoloogilised).*$

^(Kiri tundmatule).*$

^(Kiri Jenaer Allgemeine Literaturzeitung toimetusele).*$

^(Eksliibris Nikolai Birukovile).*$

^(Eksliibris Nikolai Issakovile).*$

^(\[Eksliibris Aleksandr).*$

^(WHP Cruise Summary Information of section).*$

^(Measurement of the top quark\-pair production cross section with ATLAS in pp collisions at).*$

^(Measurement of the spin\-dependent structure function).*

^(lorem ipsum).*

package eu.dnetlib.pace.clustering;

public enum Clustering {

	acronyms, ngrams, ngrampairs, suffixprefix, spacetrimmingfieldvalue

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.Map;

import java.util.Set;

import com.google.common.collect.Sets;

public class SuffixPrefix extends AbstractClusteringFunction {

	public SuffixPrefix(Map<String, Integer> params) {

		super(params);

	}

	@Override

	protected Collection<String> doApply(String s) {

		return suffixPrefix(s, param("len"), param("max"));

	}

	private Collection<String> suffixPrefix(String s, int len, int max) {

		final Set<String> bigrams = Sets.newLinkedHashSet();

		int i = 0;

		while (++i < s.length() && bigrams.size() < max) {

			int j = s.indexOf(" ", i);

			int offset = j + len + 1 < s.length() ? j + len + 1 : s.length();

			if (j - len > 0) {

				String bigram = s.substring(j - len, offset).replaceAll(" ", "").trim();

				if (bigram.length() >= 4) {

					bigrams.add(bigram);

				}

			}

		}

		return bigrams;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.List;

import java.util.Map;

import com.google.common.collect.Lists;

public class NgramPairs extends Ngrams {

	public NgramPairs(Map<String, Integer> params) {

		super(params);

	}

	@Override

	protected Collection<String> doApply(String s) {

		return ngramPairs(Lists.newArrayList(getNgrams(s, param("ngramLen"), param("max") * 2, 1, 2)), param("max"));

	}	

	private Collection<String> ngramPairs(final List<String> ngrams, int maxNgrams) {

		Collection<String> res = Lists.newArrayList();

		int j = 0;

		for (int i = 0; i < ngrams.size() && res.size() < maxNgrams; i++) {

			if (++j >= ngrams.size()) {

				break;

			}

			res.add(ngrams.get(i) + ngrams.get(j));

			//System.out.println("-- " + concatNgrams);

		}

		return res;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.List;

import java.util.Map;

import org.apache.commons.lang.RandomStringUtils;

import org.apache.commons.lang.StringUtils;

import com.google.common.collect.Lists;

public class SpaceTrimmingFieldValue extends AbstractClusteringFunction {

	public SpaceTrimmingFieldValue(final Map<String, Integer> params) {

		super(params);

	}

	@Override

	protected Collection<String> doApply(final String s) {

		final List<String> res = Lists.newArrayList();

		res.add(StringUtils.isBlank(s) ? RandomStringUtils.random(getParams().get("randomLength")) : s.toLowerCase().replaceAll("\\s+", ""));

		return res;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.List;

import java.util.Map;

import eu.dnetlib.pace.model.Field;

public interface ClusteringFunction {

	public Collection<String> apply(List<Field> fields); 

	public Map<String, Integer> getParams();

}

package eu.dnetlib.pace.clustering;

import java.util.Set;

import org.apache.commons.lang.StringUtils;

import eu.dnetlib.pace.common.AbstractPaceFunctions;

public class NGramUtils extends AbstractPaceFunctions {

	private static final int SIZE = 100;

	private static Set<String> stopwords = AbstractPaceFunctions.loadFromClasspath("/eu/dnetlib/pace/config/stopwords_en.txt");

	public static String cleanupForOrdering(String s) {

		NGramUtils utils = new NGramUtils();

		return (utils.filterStopWords(utils.normalize(s), stopwords) +  StringUtils.repeat(" ", SIZE)).substring(0, SIZE).replaceAll(" ", "");

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.Map;

public class RandomClusteringFunction extends AbstractClusteringFunction {

	public RandomClusteringFunction(Map<String, Integer> params) {

		super(params);

	}

	@Override

	protected Collection<String> doApply(String s) {

		// TODO Auto-generated method stub

		return null;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.List;

import java.util.Map;

import java.util.Map.Entry;

import java.util.Set;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import eu.dnetlib.pace.config.Config;

import eu.dnetlib.pace.model.Document;

import eu.dnetlib.pace.model.FieldListImpl;

import eu.dnetlib.pace.model.MapDocument;

public class BlacklistAwareClusteringCombiner extends ClusteringCombiner {

	public static Collection<String> filterAndCombine(final MapDocument a, final Config conf, final Map<String, List<String>> blacklists) {

		final Document filtered = new BlacklistAwareClusteringCombiner().filter(a, blacklists);

		return combine(filtered, conf);

	}

	private MapDocument filter(final MapDocument a, final Map<String, List<String>> blacklists) {

		final Map<String, FieldListImpl> filtered = Maps.newHashMap(a.getFieldMap());

		if (blacklists != null) {

			for (final Entry<String, FieldListImpl> e : filtered.entrySet()) {

				final FieldListImpl fl = new FieldListImpl();

				fl.addAll(Lists.newArrayList(Iterables.filter(e.getValue(), new FieldFilter(e.getKey(), blacklists))));

				filtered.put(e.getKey(), fl);

			}

		}

		return new MapDocument(a.getIdentifier(), filtered);

	}

	/**

	 * Tries to match the fields in the regex blacklist.

	 *

	 * @param fieldName

	 * @param value

	 * @return true if the field matches, false otherwise

	 */

	protected boolean regexMatches(final String fieldName, final String value, final Map<String, Set<String>> blacklists) {

		if (blacklists.containsKey(fieldName)) {

			for (final String regex : blacklists.get(fieldName)) {

				if (value.matches(regex)) return true;

			}

		}

		return false;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.LinkedHashSet;

import java.util.Map;

import java.util.StringTokenizer;

public class Ngrams extends AbstractClusteringFunction {

	public Ngrams(Map<String, Integer> params) {

		super(params);

	}

	@Override

	protected Collection<String> doApply(String s) {

		return getNgrams(s, param("ngramLen"), param("max"), param("maxPerToken"), param("minNgramLen"));

	}

	protected Collection<String> getNgrams(String s, int ngramLen, int max, int maxPerToken, int minNgramLen) {

		final Collection<String> ngrams = new LinkedHashSet<String>();

		final StringTokenizer st = new StringTokenizer(s);

		while (st.hasMoreTokens()) {

			final String token = st.nextToken();

			if (!token.isEmpty()) {

				for (int i = 0; i < maxPerToken && ngramLen + i <= token.length(); i++) {

					String ngram = (token + "    ").substring(i, ngramLen + i).trim();

					if (ngrams.size() >= max) {

						return ngrams;

					}

					if (ngram.length() >= minNgramLen) {

						ngrams.add(ngram);

					}

				}

			}

		}

		//System.out.println(ngrams + " n: " + ngrams.size());

		return ngrams;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.List;

import java.util.Map;

import com.google.common.collect.Sets;

import eu.dnetlib.pace.common.AbstractPaceFunctions;

import eu.dnetlib.pace.model.Field;

public abstract class AbstractClusteringFunction extends AbstractPaceFunctions implements ClusteringFunction {

	protected Map<String, Integer> params;

	public AbstractClusteringFunction(final Map<String, Integer> params) {

		this.params = params;

	}

	protected abstract Collection<String> doApply(String s);

	@Override

	public Collection<String> apply(List<Field> fields) {

		Collection<String> c = Sets.newLinkedHashSet();

		for(Field f : fields) {

			c.addAll(filterBlacklisted(doApply(filterStopWords(normalize(f.stringValue()), stopwords)), ngramBlacklist));

		}

		return c;

	}

	public Map<String, Integer> getParams() {

		return params;

	}

	protected Integer param(String name) {

		return params.get(name);

	}

}

package eu.dnetlib.pace.clustering;

import java.util.List;

import java.util.Map;

import com.google.common.base.Predicate;

import eu.dnetlib.pace.model.Field;

public class FieldFilter implements Predicate<Field> {

	private Map<String, List<String>> blacklists;

	private String filedName;

	public FieldFilter(final String fieldName, final Map<String, List<String>> blacklists) {

		this.filedName = fieldName;

		this.blacklists = blacklists;

	}

	@Override

	public boolean apply(final Field f) {

		return !regexMatches(filedName, f.stringValue(), blacklists);

	}

	/**

	 * Tries to match the fields in the regex blacklist.

	 *

	 * @param fieldName

	 * @param value

	 * @return true if the field matches, false otherwise

	 */

	protected boolean regexMatches(final String fieldName, final String value, final Map<String, List<String>> blacklists) {

		if (blacklists.containsKey(fieldName)) {

			final Iterable<String> regexes = blacklists.get(fieldName);

			for (final String regex : regexes) {

				if (value.matches(regex)) return true;

			}

		}

		return false;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.Map;

import java.util.Set;

import java.util.StringTokenizer;

import com.google.common.collect.Sets;

public class Acronyms extends AbstractClusteringFunction {

	public Acronyms(Map<String, Integer> params) {

		super(params);

	}

	@Override

	protected Collection<String> doApply(String s) {

		return extractAcronyms(s, param("max"), param("minLen"), param("maxLen"));

	}

	private Set<String> extractAcronyms(final String s, int maxAcronyms, int minLen, int maxLen) {

		final Set<String> acronyms = Sets.newLinkedHashSet();

		for (int i = 0; i < maxAcronyms; i++) {

			final StringTokenizer st = new StringTokenizer(s);

			final StringBuilder sb = new StringBuilder();

			while (st.hasMoreTokens()) {

				final String token = st.nextToken();

				if (sb.length() > maxLen) {

					break;

				}

				if (token.length() > 1 && i < token.length()) {

					sb.append(token.charAt(i));

				}

			}

			String acronym = sb.toString();

			if (acronym.length() > minLen) {

				acronyms.add(acronym);

			}

		}

		return acronyms;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.List;

import com.google.common.collect.Sets;

import eu.dnetlib.pace.config.Config;

import eu.dnetlib.pace.model.ClusteringDef;

import eu.dnetlib.pace.model.Document;

import eu.dnetlib.pace.model.FieldList;

public class ClusteringCombiner {

	public static Collection<String> combine(final Document a, final Config conf) {

		return new ClusteringCombiner().doCombine(a, conf.clusterings());

	}

	private Collection<String> doCombine(final Document a, final List<ClusteringDef> defs) {

		final Collection<String> res = Sets.newLinkedHashSet();

		for (final ClusteringDef cd : defs) {

			for (final String fieldName : cd.getFields()) {

				final FieldList values = a.values(fieldName);

				res.addAll(cd.getClusteringFunction().apply(values));

			}

		}

		return res;

	}

}

package eu.dnetlib.pace.model;

import java.util.Collection;

import java.util.Iterator;

import java.util.List;

import java.util.ListIterator;

import com.google.common.base.Function;

import com.google.common.base.Joiner;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import eu.dnetlib.pace.config.Type;

/**

 * The Class FieldListImpl.

 */

public class FieldListImpl extends AbstractField implements FieldList {

	/** The fields. */

	private List<Field> fields;

	/**

	 * Instantiates a new field list impl.

	 */

	public FieldListImpl() {

		fields = Lists.newArrayList();

	}

	/**

	 * Instantiates a new field list impl.

	 * 

	 * @param name

	 *            the name

	 */

	public FieldListImpl(final String name) {

		super(Type.List, name);

		fields = Lists.newArrayList();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#add(java.lang.Object)

	 */

	@Override

	public boolean add(final Field f) {

		return fields.add(f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#add(int, java.lang.Object)

	 */

	@Override

	public void add(final int i, final Field f) {

		fields.add(i, f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#addAll(java.util.Collection)

	 */

	@Override

	public boolean addAll(final Collection<? extends Field> f) {

		return fields.addAll(f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#addAll(int, java.util.Collection)

	 */

	@Override

	public boolean addAll(final int i, final Collection<? extends Field> f) {

		return fields.addAll(i, f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#clear()

	 */

	@Override

	public void clear() {

		fields.clear();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#contains(java.lang.Object)

	 */

	@Override

	public boolean contains(final Object o) {

		return fields.contains(o);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#containsAll(java.util.Collection)

	 */

	@Override

	public boolean containsAll(final Collection<?> f) {

		return fields.containsAll(f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#get(int)

	 */

	@Override

	public Field get(final int i) {

		return fields.get(i);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#indexOf(java.lang.Object)

	 */

	@Override

	public int indexOf(final Object o) {

		return fields.indexOf(o);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.model.Field#isEmpty()

	 */

	@Override

	public boolean isEmpty() {

		return fields.isEmpty();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.lang.Iterable#iterator()

	 */

	@Override

	public Iterator<Field> iterator() {

		return fields.iterator();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#lastIndexOf(java.lang.Object)

	 */

	@Override

	public int lastIndexOf(final Object o) {

		return fields.lastIndexOf(o);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#listIterator()

	 */

	@Override

	public ListIterator<Field> listIterator() {

		return fields.listIterator();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#listIterator(int)

	 */

	@Override

	public ListIterator<Field> listIterator(final int i) {

		return fields.listIterator(i);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#remove(java.lang.Object)

	 */

	@Override

	public boolean remove(final Object o) {

		return fields.remove(o);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#remove(int)

	 */

	@Override

	public Field remove(final int i) {

		return fields.remove(i);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#removeAll(java.util.Collection)

	 */

	@Override

	public boolean removeAll(final Collection<?> f) {

		return fields.removeAll(f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#retainAll(java.util.Collection)

	 */

	@Override

	public boolean retainAll(final Collection<?> f) {

		return fields.retainAll(f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#set(int, java.lang.Object)

	 */

	@Override

	public Field set(final int i, final Field f) {

		return fields.set(i, f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#size()

	 */

	@Override

	public int size() {

		return fields.size();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#subList(int, int)

	 */

	@Override

	public List<Field> subList(final int from, final int to) {

		return fields.subList(from, to);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#toArray()

	 */

	@Override

	public Object[] toArray() {

		return fields.toArray();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#toArray(java.lang.Object[])

	 */

	@Override

	public <T> T[] toArray(final T[] t) {

		return fields.toArray(t);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.model.Field#stringValue()

	 */

	@Override

	public String stringValue() {

		return Joiner.on(" ").join(stringList());

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.model.FieldList#stringList()

	 */

	@Override

	public List<String> stringList() {

		return Lists.newArrayList(Iterables.transform(fields, new Function<Field, String>() {

			@Override

			public String apply(final Field f) {

				return f.stringValue();

			}

		}));

	}

	@Override