D-Net

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-pace-core/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "dnet-pace-core"}

package eu.dnetlib.pace.clustering;

import java.util.Map;

import org.junit.Before;

import org.junit.Test;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import eu.dnetlib.pace.AbstractPaceTest;

public class ClusteringFunctionTest extends AbstractPaceTest {

	private Map<String, Integer> params;

	@Before

	public void setUp() throws Exception {

		params = Maps.newHashMap();

	}

	@Test

	public void testNgram() {

		params.put("ngramLen", 2);

		params.put("max", 8);

		params.put("maxPerToken", 1);

		params.put("minNgramLen", 3);

		final ClusteringFunction ngram = new Ngrams(params);

		final String s = "Search for the Standard Model Higgs Boson";

		System.out.println(s);

		System.out.println(ngram.apply(Lists.newArrayList(title(s))));

	}

	@Test

	public void testNgramPairs() {

		params.put("ngramLen", 2);

		params.put("max", 4);

		final ClusteringFunction np = new NgramPairs(params);

		final String s = "Search for the Standard Model Higgs Boson";

		System.out.println(s);

		System.out.println(np.apply(Lists.newArrayList(title(s))));

	}

	@Test

	public void testAcronym() {

		params.put("max", 4);

		params.put("minLen", 1);

		params.put("maxLen", 3);

		final ClusteringFunction acro = new Acronyms(params);

		final String s = "Search for the Standard Model Higgs Boson";

		System.out.println(s);

		System.out.println(acro.apply(Lists.newArrayList(title(s))));

	}

	@Test

	public void testSuffixPrefix() {

		params.put("len", 2);

		params.put("max", 3);

		final ClusteringFunction sp = new SuffixPrefix(params);

		final String s = "Search for the Standard Model Higgs Boson";

		System.out.println(s);

		System.out.println(sp.apply(Lists.newArrayList(title(s))));

	}

	@Test

	public void testFieldValue() {

		final ClusteringFunction sp = new SpaceTrimmingFieldValue(params);

		final String s = "Search for the Standard Model Higgs Boson";

		System.out.println(s);

		System.out.println(sp.apply(Lists.newArrayList(title(s))));

	}

}

package eu.dnetlib.pace.model;

import static org.junit.Assert.assertFalse;

import static org.junit.Assert.assertTrue;

import org.junit.Test;

public class PersonComparatorUtilsSimilarityTest {

	@Test

	public void testSimilarity_0() {

		assertTrue(PersonComparatorUtils.areSimilar("Artini Michele", "Michele Artini"));

	}

	@Test

	public void testSimilarity_1() {

		assertTrue(PersonComparatorUtils.areSimilar("ARTINI Michele", "Artini, Michele"));

	}

	@Test

	public void testSimilarity_2() {

		assertTrue(PersonComparatorUtils.areSimilar("Artini, M.", "Artini Michele"));

	}

	@Test

	public void testSimilarity_3() {

		assertTrue(PersonComparatorUtils.areSimilar("Artini, M.G.", "Artini, Michele"));

	}

	@Test

	public void testSimilarity_4() {

		assertTrue(PersonComparatorUtils.areSimilar("Artini, M.", "Artini, M.G."));

	}

	@Test

	public void testSimilarity_5() {

		assertTrue(PersonComparatorUtils.areSimilar("Artini, M. (sig.)", "Artini, Michele"));

	}

	@Test

	public void testSimilarity_6() {

		assertFalse(PersonComparatorUtils.areSimilar("Artini, M.", "Artini, G."));

	}

	@Test

	public void testSimilarity_7() {

		assertFalse(PersonComparatorUtils.areSimilar("Artini, M.G.", "Artini, M.A."));

	}

	@Test

	public void testSimilarity_8() {

		assertFalse(PersonComparatorUtils.areSimilar("Artini, M.", "Artini, Giuseppe"));

	}

	@Test

	public void testSimilarity_9() {

		assertFalse(PersonComparatorUtils.areSimilar("Manghi, Paolo", "Artini, Michele"));

	}

	@Test

	public void testSimilarity_10() {

		assertTrue(PersonComparatorUtils.areSimilar("Artini, Michele", "Artini, Michele Giovanni"));

	}

	@Test

	public void testSimilarity_11() {

		assertFalse(PersonComparatorUtils.areSimilar("Artini, M.A.G.", "Artini, M.B.G."));

	}

	@Test

	public void testSimilarity_12() {

		assertFalse(PersonComparatorUtils.areSimilar("Artini Manghi, M.", "Artini, Michele"));

	}

	@Test

	public void testSimilarity_13() {

		assertTrue(PersonComparatorUtils.areSimilar("Artini Manghi, M.", "Artini Manghi Michele"));

	}

	@Test

	public void testSimilarity_14() {

		assertFalse(PersonComparatorUtils.areSimilar("Artini, Michele", "Michele, Artini"));

	}

	@Test

	public void testSimilarity_15() {

		assertTrue(PersonComparatorUtils.areSimilar("Artini, M.", "Michele ARTINI"));

	}

}

package eu.dnetlib.pace.model;

import static org.junit.Assert.assertEquals;

import java.text.Normalizer;

import java.util.Queue;

import org.junit.Test;

import com.google.common.collect.Lists;

public class PersonTest {

	@Test

	public void test_1() {

		check("Atzori, Claudio", "Atzori, Claudio");

	}

	@Test

	public void test_2() {

		check("Atzori, Claudio A.", "Atzori, Claudio A.");

	}

	@Test

	public void test_3() {

		check("Claudio ATZORI", "Atzori, Claudio");

	}

	@Test

	public void test_4() {

		check("ATZORI, Claudio", "Atzori, Claudio");

	}

	@Test

	public void test_5() {

		check("Claudio Atzori", "Claudio Atzori");

	}

	@Test

	public void test_6() {

		check(" Manghi ,  Paolo", "Manghi, Paolo");

	}

	@Test

	public void test_7() {

		check("ATZORI, CLAUDIO", "Atzori, Claudio");

	}

	@Test

	public void test_8() {

		check("ATZORI, CLAUDIO A", "Atzori, Claudio A.");

	}

	@Test

	public void test_9() {

		check("Bølviken, B.", "Bølviken, B.");

	}

	@Test

	public void test_10() {

		check("Bñlviken, B.", "B" + Normalizer.normalize("ñ", Normalizer.Form.NFD) + "lviken, B.");

	}

	@Test

	public void test_11() {

		check("aáeéiíoóöőuúüű AÁEÉIÍOÓÖŐUÚÜŰ ø", "Aaeeiioooouuuu, Aaeeiioooouuuu Ø.", true);

	}

	@Test

	public void test_12() {

		check("aáeéiíoóöőuúüű AÁEÉIÍOÓÖŐUÚÜŰz ø", Normalizer.normalize("aáeéiíoóöőuúüű AÁEÉIÍOÓÖŐUÚÜŰz ø", Normalizer.Form.NFD), false);

	}

	@Test

	public void test_13() {

		check("Tkačíková, Daniela", Normalizer.normalize("Tkačíková, Daniela", Normalizer.Form.NFD), false);

	}

	@Test

	public void test_hashes() {

		checkHash(" Claudio  ATZORI ", "ATZORI Claudio", "Atzori , Claudio", "ATZORI, Claudio");

	}

	private void checkHash(String... ss) {

		Queue<String> q = Lists.newLinkedList(Lists.newArrayList(ss));

		String h1 = new Person(q.remove(), false).hash();

		while (!q.isEmpty()) {

			assertEquals(h1, new Person(q.remove(), false).hash());

		}

	}

	private void check(String s, String expectedFullName) {

		check(s, expectedFullName, false);

	}

	private void check(String s, String expectedFullName, boolean aggressive) {

		Person p = new Person(s, aggressive);

		System.out.println("original:   " + p.getOriginal());

		System.out.println("accurate:   " + p.isAccurate());

		System.out.println("normalised: '" + p.getNormalisedFullname() + "'");

		if (p.isAccurate()) {

			System.out.println("name:     " + p.getNormalisedFirstName());

			System.out.println("surname:  " + p.getNormalisedSurname());

		}

		System.out.println("hash: " + p.hash());

		System.out.println("");

		assertEquals(expectedFullName, p.getNormalisedFullname());

	}

}

package eu.dnetlib.pace.model;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertTrue;

import java.util.Set;

import org.junit.Test;

public class PersonComparatorUtilsNGramsTest {

	@Test

	public void testNormaizePerson_1() {

		verifyGetNgramsForPerson("Artini Michele", 2, "a_michele", "m_artini");

	}

	@Test

	public void testNormaizePerson_2() {

		verifyGetNgramsForPerson("Michele Artini", 2, "a_michele", "m_artini");

	}

	@Test

	public void testNormaizePerson_3() {

		verifyGetNgramsForPerson("Michele ARTINI", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_4() {

		verifyGetNgramsForPerson("ARTINI Michele", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_5() {

		verifyGetNgramsForPerson("Michele G. Artini", 2, "m_artini", "g_artini");

	}

	@Test

	public void testNormaizePerson_6() {

		verifyGetNgramsForPerson(" Artini, Michele ", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_7() {

		verifyGetNgramsForPerson("Artini, Michele (sig.)", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_8() {

		verifyGetNgramsForPerson("Artini Michele [sig.] ", 2, "a_michele", "m_artini");

	}

	@Test

	public void testNormaizePerson_9() {

		verifyGetNgramsForPerson("Artini, M", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_10() {

		verifyGetNgramsForPerson("Artini, M.", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_11() {

		verifyGetNgramsForPerson("Artini, M. (sig.)", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_12() {

		verifyGetNgramsForPerson("Artini, M[sig.] ", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_13() {

		verifyGetNgramsForPerson("Artini-SIG, Michele ", 1, "m_artini-sig");

	}

	@Test

	public void testNormaizePerson_14() {

		verifyGetNgramsForPerson("Artini - SIG, Michele ", 1, "m_artini-sig");

	}

	@Test

	public void testNormaizePerson_15() {

		verifyGetNgramsForPerson("Artini {sig.}, M", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_16() {

		verifyGetNgramsForPerson("Artini, M., sig.", 1, "m_artini");

	}

	@Test

	public void testNormaizePerson_17() {

		verifyGetNgramsForPerson("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA, BBBBBBBBBBBBBBBBBBBBBBBBBBBBB CCCCCCCCCCCCCCCCCCCC", 0);

	}

	@Test

	public void testNormaizePerson_18() {

		verifyGetNgramsForPerson("Dell'amico, Andrea", 1, "a_amico");

	}

	@Test

	public void testNormaizePerson_19() {

		verifyGetNgramsForPerson("Smith, Paul van der", 1, "p_smith");

	}

	@Test

	public void testNormaizePerson_20() {

		verifyGetNgramsForPerson("AAAAAAA, BBBB, CCCC, DDDD, EEEE", 1, "b_aaaaaaa");

	}

	@Test

	public void testNormaizePerson_21() {

		verifyGetNgramsForPerson("Kompetenzzentrum Informelle Bildung (KIB),", 6);

	}

	private void verifyGetNgramsForPerson(String name, int expectedSize, String... expectedTokens) {

		Set<String> list = PersonComparatorUtils.getNgramsForPerson(name);

		System.out.println(list);

		assertEquals(expectedSize, list.size());

		for (String s : expectedTokens) {

			assertTrue(list.contains(s));

		}

	}

}

package eu.dnetlib.pace.config;

import static org.junit.Assert.assertFalse;

import static org.junit.Assert.assertNotNull;

import static org.junit.Assert.assertTrue;

import java.util.List;

import java.util.Map;

import org.junit.Test;

import eu.dnetlib.pace.model.ClusteringDef;

import eu.dnetlib.pace.model.CondDef;

import eu.dnetlib.pace.model.FieldDef;

public class ConfigTest {

	public static String cfgFull = "pace.conf { " + "clustering { " + "acronyms     { fields = [title], params = { max = 1, minLen = 2, maxLen = 4} }, "

			+ "ngrampairs   { fields = [title], params = { max = 1, ngramLen = 3} }, " + "suffixprefix { fields = [title], params = { max = 1, len = 3 } } }, "

			+

			"conditions { " + "yearMatch { fields = [dateofacceptance] }, " + "titleVersionMatch { fields = [title] } }," +

			"model { " + "title/value { algo = JaroWinkler, type = String, weight = 1.0, ignoreMissing = false }, "

			+ "dateofacceptance { algo = Null, type = String, weight = 0.0, ignoreMissing = true } }, " +

			"blacklists = {" + "title = [" + "\"^(Corpus Oral Dialectal \\\\(COD\\\\).).*$\", " + "\"^(Kiri Karl Morgensternile).*$\", "

			+ "\"^(Kiri A. de Vignolles).*$\", " + "\"^(\\\\[Eksliibris Aleksandr).*\\\\]$\", " + "\"^(\\\\[Eksliibris Aleksandr).*$\", "

			+ "\"^(Eksliibris Aleksandr).*$\"" + "] }" + "}";

	public static String cfgOrganizations = "pace.conf { " + "clustering { " + "ngrampairs { fields = [legalname], params = { max = 1, ngramLen = 3} }, "

			+ "suffixprefix { fields = [legalname], params = { max = 1, len = 3 } } }, " + "model { "

			+ "legalname 	   { algo = JaroWinkler, type = String, weight = 0.6, ignoreMissing = false }, "

			+ "legalshortname { algo = JaroWinkler, type = String, weight = 0.4, ignoreMissing = true  } } } ";

	private Config config;

	@Test

	public void testFull() {

		config = DynConf.load(cfgFull);

		assertNotNull(config);

		checkFields(config.fields());

		checkConditions(config.conditions());

		checkClustering(config.clusterings());

		checkBlacklists(config.blacklists());

	}

	@Test

	public void testOrganizations() {

		config = DynConf.load(cfgOrganizations);

		assertNotNull(config);

		checkFields(config.fields());

		checkClustering(config.clusterings());

		assertNotNull(config.blacklists());

		assertTrue(config.blacklists().isEmpty());

	}

	private void checkBlacklists(final Map<String, Iterable<String>> b) {

		assertNotNull(b);

		assertFalse(b.isEmpty());

		System.out.println("blacklists:\n" + b);

		assertTrue(tryMatch(b.get("title"), "Corpus Oral Dialectal (COD). Barcelona"));

		assertTrue(tryMatch(b.get("title"), "Kiri Karl Morgensternile"));

		assertTrue(tryMatch(b.get("title"), "[Eksliibris Aleksandr Fadejevile]"));

		assertTrue(tryMatch(b.get("title"), "Eksliibris Aleksandr Baldinile"));

		assertTrue(tryMatch(b.get("title"), "Kiri A. de Vignolles'ile, Potsdam"));

		assertTrue(tryMatch(b.get("title"), "Kiri A. de Vignolles`ile, Brandenburg"));

	}

	private boolean tryMatch(final Iterable<String> set, final String string) {

		for (String regex : set) {

			if (string.matches(regex)) { return true; }

		}

		return false;

	}

	private void checkClustering(final List<ClusteringDef> clusterings) {

		assertNotNull(clusterings);

		for (ClusteringDef cd : clusterings) {

			assertNotNull(cd);

			assertNotNull(cd.getClusteringFunction());

			assertNotNull(cd.getClusteringFunction().getParams());

			assertNotNull(cd.getFields());

			assertFalse(cd.getFields().isEmpty());

		}

		System.out.println("clustering: " + clusterings);

	}

	private void checkConditions(final List<CondDef> conds) {

		assertNotNull(conds);

		for (CondDef cd : conds) {

			assertNotNull(cd);

			assertNotNull(cd.getConditionAlgo());

		}

		System.out.println("conditions: " + conds);

	}

	private void checkFields(final List<FieldDef> fields) {

		assertNotNull(fields);

		for (FieldDef fd : fields) {

			assertNotNull(fd);

			assertNotNull(fd.getName());

			assertTrue(fd.getName().length() > 0);

			assertNotNull(fd.getAlgo());

			System.out.println(fd);

		}

	}

}

package eu.dnetlib.pace.distance;

import org.junit.Before;

import org.junit.Test;

import eu.dnetlib.pace.common.AbstractPaceFunctions;

public class DistanceAlgoTest extends AbstractPaceFunctions {

	private final static String TEST_STRING = "Toshiba NB550D: è un netbook su piattaforma AMD Fusion⁽¹²⁾.";

	@Before

	public void setup() {

		System.out.println("****************************************************************");

		System.out.println("Test String    : " + TEST_STRING);

	}

	@Test

	public void testGetNumbers() {

		System.out.println("Numbers        : " + getNumbers(TEST_STRING));

	}

	@Test

	public void testRemoveSymbols() {

		System.out.println("Without symbols: " + removeSymbols(TEST_STRING));

	}

	@Test

	public void testFixAliases() {

		System.out.println("Fixed aliases  : " + fixAliases(TEST_STRING));

	}

	@Test

	public void testCleanup() {

		System.out.println("cleaned up     : " + cleanup(TEST_STRING));

	}

}

package eu.dnetlib.pace;

import java.io.IOException;

import java.io.StringWriter;

import org.apache.commons.io.IOUtils;

import eu.dnetlib.pace.config.Config;

import eu.dnetlib.pace.config.DynConf;

import eu.dnetlib.pace.config.Type;

import eu.dnetlib.pace.model.Field;

import eu.dnetlib.pace.model.FieldValueImpl;

public abstract class AbstractPaceTest {

	protected Config getResultConf() {

		return DynConf.load(readFromClasspath("/eu/dnetlib/pace/config/result.pace.conf"));

	}

	protected Config getOrganizationConf() {

		return DynConf.load(readFromClasspath("/eu/dnetlib/pace/config/organization.pace.conf"));

	}

	private String readFromClasspath(final String filename) {

		StringWriter sw = new StringWriter();

		try {

			IOUtils.copy(getClass().getResourceAsStream(filename), sw);

			return sw.toString();

		} catch (IOException e) {

			throw new RuntimeException("cannot load resource from classpath: " + filename);

		}

	}

	protected Field title(final String s) {

		return new FieldValueImpl(Type.String, "title", s);

	}

}

package eu.dnetlib.pace.util;

import static org.junit.Assert.assertFalse;

import static org.junit.Assert.assertNotNull;

import static org.junit.Assert.assertTrue;

import java.util.HashSet;

import org.junit.Test;

import com.google.common.collect.Lists;

import com.google.common.collect.Sets;

public class DedupConfigTest {

	@Test

	public void testCfg1() {

		final String s =

				"dedup.conf { " +

						"id = '01', " +

						"threshold = 0.99, " +

						"run = '001', " +

						"entity.type = organization, " +

						"order.field = legalname, " +

						"ngram.fields = [legalname], " +

						"queue.max.size = 2000, " +

						"group.max.size = 10, " +

						"sliding.window.size = 200, " +

						"rootbuilder = [organization,projectOrganization_participation_isParticipant,datasourceOrganization_provision_isProvidedBy], " +

						"skiplist = [od_______908,od________18]," +

						"include.children = true }";

		final DedupConfig cfg = DedupConfigLoader.load(s);

		assertNotNull(cfg);

		assertTrue(cfg.getConfigurationId().equals("01"));

		assertTrue(cfg.getThreshold() == 0.99);

		assertTrue(cfg.getDedupRun().equals("001"));

		assertTrue(cfg.getEntityType().equals("organization"));

		assertTrue(cfg.getOrderField().equals("legalname"));

		assertTrue(Lists.newArrayList("organization", "projectOrganization_participation_isParticipant", "datasourceOrganization_provision_isProvidedBy")

				.equals(cfg.getRootBuilderFamilies()));

		assertTrue(Sets.newHashSet("od_______908", "od________18").equals(cfg.getSkipList()));

		assertTrue(cfg.getSlidingWindowSize() == 200);

		assertTrue(cfg.getGroupMaxSize() == 10);

		assertTrue(cfg.getQueueMaxSize() == 2000);

		assertTrue(cfg.isIncludeChildren());

	}

	@Test

	public void testCfg2() {

		final String s =

				"dedup.conf { " +

						"id = '01', " +

						"threshold = 0.99, " +

						"run = '001', " +

						"entity.type = organization, " +

						"order.field = legalname, " +

						"ngram.fields = [legalname], " +

						"queue.max.size = 2000, " +

						"group.max.size = 10, " +

						"sliding.window.size = 200, " +

						"rootbuilder = [organization,projectOrganization_participation_isParticipant,datasourceOrganization_provision_isProvidedBy], " +

						"include.children = false }";

		final DedupConfig cfg = DedupConfigLoader.load(s);

		assertNotNull(cfg);

		assertTrue(cfg.getConfigurationId().equals("01"));

		assertTrue(cfg.getThreshold() == 0.99);

		assertTrue(cfg.getEntityType().equals("organization"));

		assertTrue(cfg.getOrderField().equals("legalname"));

		assertTrue(Lists.newArrayList("organization", "projectOrganization_participation_isParticipant", "datasourceOrganization_provision_isProvidedBy")

				.equals(cfg.getRootBuilderFamilies()));

		assertTrue(new HashSet<String>().equals(cfg.getSkipList()));

		assertTrue(cfg.getSlidingWindowSize() == 200);

		assertTrue(cfg.getGroupMaxSize() == 10);

		assertTrue(cfg.getQueueMaxSize() == 2000);

		assertFalse(cfg.isIncludeChildren());

	}

}

pace.conf { 

	clustering {

		fieldvalue { fields = [legalshortname], params = { } }

	},

	conditions { },

	model {

		legalname { algo = JaroWinkler, type = String, weight = 1.0, ignoreMissing = false, path = organization/metadata/legalname/value }

	} 

}

^(Corpus Oral Dialectal \(COD\)\.).*$

^(Kiri Karl Morgensternile).*$

^(\[Eksliibris Aleksandr).*\]$

^(Kiri A\. de Vignolles).*$

^(2 kirja Karl Morgensternile).*$

^(Pirita kloostri idaosa arheoloogilised).*$

^(Kiri tundmatule).*$

^(Kiri Jenaer Allgemeine Literaturzeitung toimetusele).*$

^(Eksliibris Nikolai Birukovile).*$

^(Eksliibris Nikolai Issakovile).*$

^(\[Eksliibris Aleksandr).*$

^(WHP Cruise Summary Information of section).*$

^(Measurement of the top quark\-pair production cross section with ATLAS in pp collisions at).*$

^(Measurement of the spin\-dependent structure function).*

^(lorem ipsum).*

pace.conf {

	clustering {

		acronyms { fields = [title], params = { max = 1, minLen = 2, maxLen = 4} },

		ngrampairs { fields = [title], params = { max = 1, ngramLen = 3} },

		suffixprefix { fields = [title], params = { max = 1, len = 3 } } 

	},

	strictconditions {

		exactMatch { fields = [pid] }

	}, 

	conditions { 

		yearMatch { fields = [dateofacceptance] },

		titleVersionMatch { fields = [title] },

		sizeMatch { fields = [authors] } 

	},

	model {

		pid { algo = ExactMatch, type = String, weight = 0.0, ignoreMissing = true, path = pid/value, overrideMatch = true }, 	

		title { algo = JaroWinkler, type = String, weight = 1.0, ignoreMissing = false, path = result/metadata/title/value },

		dateofacceptance { algo = Null, type = String, weight = 0.0, ignoreMissing = true, path = result/metadata/dateofacceptance/value } ,

		authors { algo = Null, type = List, weight = 0.0, ignoreMissing = true, path = result/author/metadata/fullname/value } 		

	},

	blacklists = {

		title = [

			"^(Corpus Oral Dialectal \\(COD\\)\\.).*$",

			"^(Kiri Karl Morgensternile).*$",

			"^(\\[Eksliibris Aleksandr).*\\]$",

			"^(\\[Eksliibris Aleksandr).*$",

			"^(Eksliibris Aleksandr).*$",

			"^(Kiri A\\. de Vignolles).*$",

			"^(2 kirja Karl Morgensternile).*$",

			"^(Pirita kloostri idaosa arheoloogilised).*$",

			"^(Kiri tundmatule).*$",

			"^(Kiri Jenaer Allgemeine Literaturzeitung toimetusele).*$",

			"^(Eksliibris Nikolai Birukovile).*$",

			"^(Eksliibris Nikolai Issakovile).*$",

			"^(WHP Cruise Summary Information of section).*$",

			"^(Measurement of the top quark\\-pair production cross section with ATLAS in pp collisions at).*$",

			"^(Measurement of the spin\\-dependent structure function).*"

		] } 

}

package eu.dnetlib.pace.clustering;

public enum Clustering {

	acronyms, ngrams, ngrampairs, suffixprefix, spacetrimmingfieldvalue

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.Map;

import java.util.Set;

import com.google.common.collect.Sets;

public class SuffixPrefix extends AbstractClusteringFunction {

	public SuffixPrefix(Map<String, Integer> params) {

		super(params);

	}

	@Override

	protected Collection<String> doApply(String s) {

		return suffixPrefix(s, param("len"), param("max"));

	}

	private Collection<String> suffixPrefix(String s, int len, int max) {

		final Set<String> bigrams = Sets.newLinkedHashSet();

		int i = 0;

		while (++i < s.length() && bigrams.size() < max) {

			int j = s.indexOf(" ", i);

			int offset = j + len + 1 < s.length() ? j + len + 1 : s.length();

			if (j - len > 0) {

				String bigram = s.substring(j - len, offset).replaceAll(" ", "").trim();

				if (bigram.length() >= 4) {

					bigrams.add(bigram);

				}

			}

		}

		return bigrams;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.List;

import java.util.Map;

import com.google.common.collect.Lists;

public class NgramPairs extends Ngrams {

	public NgramPairs(Map<String, Integer> params) {

		super(params);

	}

	@Override

	protected Collection<String> doApply(String s) {

		return ngramPairs(Lists.newArrayList(getNgrams(s, param("ngramLen"), param("max") * 2, 1, 2)), param("max"));

	}	

	private Collection<String> ngramPairs(final List<String> ngrams, int maxNgrams) {

		Collection<String> res = Lists.newArrayList();

		int j = 0;

		for (int i = 0; i < ngrams.size() && res.size() < maxNgrams; i++) {

			if (++j >= ngrams.size()) {

				break;

			}

			res.add(ngrams.get(i) + ngrams.get(j));

			//System.out.println("-- " + concatNgrams);

		}

		return res;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.List;

import java.util.Map;

import org.apache.commons.lang.RandomStringUtils;

import org.apache.commons.lang.StringUtils;

import com.google.common.collect.Lists;

public class SpaceTrimmingFieldValue extends AbstractClusteringFunction {

	public SpaceTrimmingFieldValue(final Map<String, Integer> params) {

		super(params);

	}

	@Override

	protected Collection<String> doApply(final String s) {

		final List<String> res = Lists.newArrayList();

		res.add(StringUtils.isBlank(s) ? RandomStringUtils.random(getParams().get("randomLength")) : s.toLowerCase().replaceAll("\\s+", ""));

		return res;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.List;

import java.util.Map;

import eu.dnetlib.pace.model.Field;

public interface ClusteringFunction {

	public Collection<String> apply(List<Field> fields); 

	public Map<String, Integer> getParams();

}

package eu.dnetlib.pace.clustering;

import java.util.Set;

import org.apache.commons.lang.StringUtils;

import eu.dnetlib.pace.common.AbstractPaceFunctions;

public class NGramUtils extends AbstractPaceFunctions {

	private static final int SIZE = 100;

	private static Set<String> stopwords = AbstractPaceFunctions.loadFromClasspath("/eu/dnetlib/pace/config/stopwords_en.txt");

	public static String cleanupForOrdering(String s) {

		NGramUtils utils = new NGramUtils();

		return (utils.filterStopWords(utils.normalize(s), stopwords) +  StringUtils.repeat(" ", SIZE)).substring(0, SIZE).replaceAll(" ", "");

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.Map;

public class RandomClusteringFunction extends AbstractClusteringFunction {

	public RandomClusteringFunction(Map<String, Integer> params) {

		super(params);

	}

	@Override

	protected Collection<String> doApply(String s) {

		// TODO Auto-generated method stub

		return null;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.Map;

import java.util.Map.Entry;

import java.util.Set;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import eu.dnetlib.pace.config.Config;

import eu.dnetlib.pace.model.Document;

import eu.dnetlib.pace.model.FieldListImpl;

import eu.dnetlib.pace.model.MapDocument;

public class BlacklistAwareClusteringCombiner extends ClusteringCombiner {

	public static Collection<String> filterAndCombine(final MapDocument a, final Config conf, final Map<String, Iterable<String>> blacklists) {

		Document filtered = new BlacklistAwareClusteringCombiner().filter(a, blacklists);

		return combine(filtered, conf);

	}

	private MapDocument filter(final MapDocument a, final Map<String, Iterable<String>> blacklists) {

		final Map<String, FieldListImpl> filtered = Maps.newHashMap(a.getFieldMap());

		if (blacklists != null) {

			for (final Entry<String, FieldListImpl> e : filtered.entrySet()) {

				FieldListImpl fl = new FieldListImpl();

				fl.addAll(Lists.newArrayList(Iterables.filter(e.getValue(), new FieldFilter(e.getKey(), blacklists))));

				filtered.put(e.getKey(), fl);

			}

		}

		return new MapDocument(a.getIdentifier(), filtered);

	}

	/**

	 * Tries to match the fields in the regex blacklist.

	 *

	 * @param fieldName

	 * @param value

	 * @return true if the field matches, false otherwise

	 */

	protected boolean regexMatches(final String fieldName, final String value, final Map<String, Set<String>> blacklists) {

		if (blacklists.containsKey(fieldName)) {

			for (final String regex : blacklists.get(fieldName)) {

				if (value.matches(regex)) { return true; }

			}

		}

		return false;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.LinkedHashSet;

import java.util.Map;

import java.util.StringTokenizer;

public class Ngrams extends AbstractClusteringFunction {

	public Ngrams(Map<String, Integer> params) {

		super(params);

	}

	@Override

	protected Collection<String> doApply(String s) {

		return getNgrams(s, param("ngramLen"), param("max"), param("maxPerToken"), param("minNgramLen"));

	}

	protected Collection<String> getNgrams(String s, int ngramLen, int max, int maxPerToken, int minNgramLen) {

		final Collection<String> ngrams = new LinkedHashSet<String>();

		final StringTokenizer st = new StringTokenizer(s);

		while (st.hasMoreTokens()) {

			final String token = st.nextToken();

			if (!token.isEmpty()) {

				for (int i = 0; i < maxPerToken && ngramLen + i <= token.length(); i++) {

					String ngram = (token + "    ").substring(i, ngramLen + i).trim();

					if (ngrams.size() >= max) {

						return ngrams;

					}

					if (ngram.length() >= minNgramLen) {

						ngrams.add(ngram);

					}

				}

			}

		}

		//System.out.println(ngrams + " n: " + ngrams.size());

		return ngrams;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.List;

import java.util.Map;

import com.google.common.collect.Sets;

import eu.dnetlib.pace.common.AbstractPaceFunctions;

import eu.dnetlib.pace.model.Field;

public abstract class AbstractClusteringFunction extends AbstractPaceFunctions implements ClusteringFunction {

	protected Map<String, Integer> params;

	public AbstractClusteringFunction(final Map<String, Integer> params) {

		this.params = params;

	}

	protected abstract Collection<String> doApply(String s);

	@Override

	public Collection<String> apply(List<Field> fields) {

		Collection<String> c = Sets.newLinkedHashSet();

		for(Field f : fields) {

			c.addAll(filterBlacklisted(doApply(filterStopWords(normalize(f.stringValue()), stopwords)), ngramBlacklist));

		}

		return c;

	}

	public Map<String, Integer> getParams() {

		return params;

	}

	protected Integer param(String name) {

		return params.get(name);

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Map;

import com.google.common.base.Predicate;

import eu.dnetlib.pace.model.Field;

public class FieldFilter implements Predicate<Field> {

	private Map<String, Iterable<String>> blacklists;

	private String filedName;

	public FieldFilter(final String fieldName, final Map<String, Iterable<String>> blacklists) {

		this.filedName = fieldName;

		this.blacklists = blacklists;

	}

	@Override

	public boolean apply(final Field f) {

		return !regexMatches(filedName, f.stringValue(), blacklists);

	}

	/**

	 * Tries to match the fields in the regex blacklist.

	 * 

	 * @param fieldName

	 * @param value

	 * @return true if the field matches, false otherwise

	 */

	protected boolean regexMatches(final String fieldName, final String value, final Map<String, Iterable<String>> blacklists) {

		if (blacklists.containsKey(fieldName)) {

			final Iterable<String> regexes = blacklists.get(fieldName);

			for (final String regex : regexes) {

				if (value.matches(regex)) { return true; }

			}

		}

		return false;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.Map;

import java.util.Set;

import java.util.StringTokenizer;

import com.google.common.collect.Sets;

public class Acronyms extends AbstractClusteringFunction {

	public Acronyms(Map<String, Integer> params) {

		super(params);

	}

	@Override

	protected Collection<String> doApply(String s) {

		return extractAcronyms(s, param("max"), param("minLen"), param("maxLen"));

	}

	private Set<String> extractAcronyms(final String s, int maxAcronyms, int minLen, int maxLen) {

		final Set<String> acronyms = Sets.newLinkedHashSet();

		for (int i = 0; i < maxAcronyms; i++) {

			final StringTokenizer st = new StringTokenizer(s);

			final StringBuilder sb = new StringBuilder();

			while (st.hasMoreTokens()) {

				final String token = st.nextToken();

				if (sb.length() > maxLen) {

					break;

				}

				if (token.length() > 1 && i < token.length()) {

					sb.append(token.charAt(i));

				}

			}

			String acronym = sb.toString();

			if (acronym.length() > minLen) {

				acronyms.add(acronym);

			}

		}

		return acronyms;

	}

}

package eu.dnetlib.pace.clustering;

import java.util.Collection;

import java.util.List;

import com.google.common.collect.Sets;

import eu.dnetlib.pace.config.Config;

import eu.dnetlib.pace.model.ClusteringDef;

import eu.dnetlib.pace.model.Document;

public class ClusteringCombiner {

	public static Collection<String> combine(Document a, Config conf) {

		return new ClusteringCombiner().doCombine(a, conf.clusterings());

	}

	private Collection<String> doCombine(Document a, List<ClusteringDef> defs) {

		final Collection<String> res = Sets.newLinkedHashSet();

		for(ClusteringDef cd : defs) {

			for(String fieldName : cd .getFields()) {

				res.addAll(cd.getClusteringFunction().apply(a.values(fieldName)));

			}

		}

		return res;

	}

}

package eu.dnetlib.pace.model;

import java.util.Collection;

import java.util.Iterator;

import java.util.List;

import java.util.ListIterator;

import com.google.common.base.Function;

import com.google.common.base.Joiner;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import eu.dnetlib.pace.config.Type;

/**

 * The Class FieldListImpl.

 */

public class FieldListImpl extends AbstractField implements FieldList {

	/** The fields. */

	private List<Field> fields;

	/**

	 * Instantiates a new field list impl.

	 */

	public FieldListImpl() {

		fields = Lists.newArrayList();

	}

	/**

	 * Instantiates a new field list impl.

	 * 

	 * @param name

	 *            the name

	 */

	public FieldListImpl(final String name) {

		super(Type.List, name);

		fields = Lists.newArrayList();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#add(java.lang.Object)

	 */

	@Override

	public boolean add(final Field f) {

		return fields.add(f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#add(int, java.lang.Object)

	 */

	@Override

	public void add(final int i, final Field f) {

		fields.add(i, f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#addAll(java.util.Collection)

	 */

	@Override

	public boolean addAll(final Collection<? extends Field> f) {

		return fields.addAll(f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#addAll(int, java.util.Collection)

	 */

	@Override

	public boolean addAll(final int i, final Collection<? extends Field> f) {

		return fields.addAll(i, f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#clear()

	 */

	@Override

	public void clear() {

		fields.clear();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#contains(java.lang.Object)

	 */

	@Override