D-Net

import java.io.IOException;

import java.io.StringWriter;

import org.apache.commons.io.IOUtils;

import eu.dnetlib.pace.config.Config;

import eu.dnetlib.pace.config.DynConf;

import eu.dnetlib.pace.model.FieldValueImpl;

	protected Config getResultConf() {

		return DynConf.load(readFromClasspath("/eu/dnetlib/pace/config/result.pace.conf"));

	}

	protected Config getOrganizationConf() {

		return DynConf.load(readFromClasspath("/eu/dnetlib/pace/config/organization.pace.conf"));

	}

	private String readFromClasspath(final String filename) {

		StringWriter sw = new StringWriter();

		try {

			IOUtils.copy(getClass().getResourceAsStream(filename), sw);

			return sw.toString();

		} catch (IOException e) {

			throw new RuntimeException("cannot load resource from classpath: " + filename);

		}

	}

		return new FieldValueImpl(Type.String, "title", s);

pace.conf { 

	conditions { },

	model {

		legalname { algo = JaroWinkler, type = String, weight = 0.6, ignoreMissing = false, path = organization/metadata/legalname/value }, 

		legalshortname { algo = JaroWinkler, type = String, weight = 0.4, ignoreMissing = true, path = organization/metadata/legalshortname/value } 

	} 

}

pace.conf {

	clustering {

		acronyms { fields = [title], params = { max = 1, minLen = 2, maxLen = 4} },

		ngrampairs { fields = [title], params = { max = 1, ngramLen = 3} },

		suffixprefix { fields = [title], params = { max = 1, len = 3 } } 

	},

	strictconditions {

		exactMatch { fields = [pid] }

	}, 

	conditions { 

		yearMatch { fields = [dateofacceptance] },

		titleVersionMatch { fields = [title] },

		sizeMatch { fields = [authors] } 

	},

	model {

		pid { algo = ExactMatch, type = String, weight = 0.0, ignoreMissing = true, path = pid/value, overrideMatch = true }, 	

		title { algo = JaroWinkler, type = String, weight = 1.0, ignoreMissing = false, path = result/metadata/title/value },

		dateofacceptance { algo = Null, type = String, weight = 0.0, ignoreMissing = true, path = result/metadata/dateofacceptance/value } ,

		authors { algo = Null, type = List, weight = 0.0, ignoreMissing = true, path = result/author/metadata/fullname/value } 		

	},

	blacklists = {

		title = [

			"^(Corpus Oral Dialectal \\(COD\\)\\.).*$",

			"^(Kiri Karl Morgensternile).*$",

			"^(\\[Eksliibris Aleksandr).*\\]$",

			"^(\\[Eksliibris Aleksandr).*$",

			"^(Eksliibris Aleksandr).*$",

			"^(Kiri A\\. de Vignolles).*$",

			"^(2 kirja Karl Morgensternile).*$",

			"^(Pirita kloostri idaosa arheoloogilised).*$",

			"^(Kiri tundmatule).*$",

			"^(Kiri Jenaer Allgemeine Literaturzeitung toimetusele).*$",

			"^(Eksliibris Nikolai Birukovile).*$",

			"^(Eksliibris Nikolai Issakovile).*$",

			"^(WHP Cruise Summary Information of section).*$",

			"^(Measurement of the top quark\\-pair production cross section with ATLAS in pp collisions at).*$",

			"^(Measurement of the spin\\-dependent structure function).*"

		] } 

}

import eu.dnetlib.pace.model.FieldListImpl;

		final Map<String, FieldListImpl> filtered = Maps.newHashMap(a.getFieldMap());

			for (final Entry<String, FieldListImpl> e : filtered.entrySet()) {

				FieldListImpl fl = new FieldListImpl();

				fl.addAll(Lists.newArrayList(Iterables.filter(e.getValue(), new FieldFilter(e.getKey(), blacklists))));

				filtered.put(e.getKey(), fl);

	 *

package eu.dnetlib.pace.model;

import java.util.Collection;

import java.util.Iterator;

import java.util.List;

import java.util.ListIterator;

import com.google.common.base.Function;

import com.google.common.base.Joiner;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import eu.dnetlib.pace.config.Type;

/**

 * The Class FieldListImpl.

 */

public class FieldListImpl extends AbstractField implements FieldList {

	/** The fields. */

	private List<Field> fields;

	/**

	 * Instantiates a new field list impl.

	 */

	public FieldListImpl() {

		fields = Lists.newArrayList();

	}

	/**

	 * Instantiates a new field list impl.

	 * 

	 * @param name

	 *            the name

	 */

	public FieldListImpl(final String name) {

		super(Type.List, name);

		fields = Lists.newArrayList();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#add(java.lang.Object)

	 */

	@Override

	public boolean add(final Field f) {

		return fields.add(f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#add(int, java.lang.Object)

	 */

	@Override

	public void add(final int i, final Field f) {

		fields.add(i, f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#addAll(java.util.Collection)

	 */

	@Override

	public boolean addAll(final Collection<? extends Field> f) {

		return fields.addAll(f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#addAll(int, java.util.Collection)

	 */

	@Override

	public boolean addAll(final int i, final Collection<? extends Field> f) {

		return fields.addAll(i, f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#clear()

	 */

	@Override

	public void clear() {

		fields.clear();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#contains(java.lang.Object)

	 */

	@Override

	public boolean contains(final Object o) {

		return fields.contains(o);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#containsAll(java.util.Collection)

	 */

	@Override

	public boolean containsAll(final Collection<?> f) {

		return fields.containsAll(f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#get(int)

	 */

	@Override

	public Field get(final int i) {

		return fields.get(i);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#indexOf(java.lang.Object)

	 */

	@Override

	public int indexOf(final Object o) {

		return fields.indexOf(o);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.model.Field#isEmpty()

	 */

	@Override

	public boolean isEmpty() {

		return fields.isEmpty();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.lang.Iterable#iterator()

	 */

	@Override

	public Iterator<Field> iterator() {

		return fields.iterator();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#lastIndexOf(java.lang.Object)

	 */

	@Override

	public int lastIndexOf(final Object o) {

		return fields.lastIndexOf(o);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#listIterator()

	 */

	@Override

	public ListIterator<Field> listIterator() {

		return fields.listIterator();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#listIterator(int)

	 */

	@Override

	public ListIterator<Field> listIterator(final int i) {

		return fields.listIterator(i);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#remove(java.lang.Object)

	 */

	@Override

	public boolean remove(final Object o) {

		return fields.remove(o);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#remove(int)

	 */

	@Override

	public Field remove(final int i) {

		return fields.remove(i);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#removeAll(java.util.Collection)

	 */

	@Override

	public boolean removeAll(final Collection<?> f) {

		return fields.removeAll(f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#retainAll(java.util.Collection)

	 */

	@Override

	public boolean retainAll(final Collection<?> f) {

		return fields.retainAll(f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#set(int, java.lang.Object)

	 */

	@Override

	public Field set(final int i, final Field f) {

		return fields.set(i, f);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#size()

	 */

	@Override

	public int size() {

		return fields.size();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#subList(int, int)

	 */

	@Override

	public List<Field> subList(final int from, final int to) {

		return fields.subList(from, to);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#toArray()

	 */

	@Override

	public Object[] toArray() {

		return fields.toArray();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.List#toArray(java.lang.Object[])

	 */

	@Override

	public <T> T[] toArray(final T[] t) {

		return fields.toArray(t);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.model.Field#stringValue()

	 */

	@Override

	public String stringValue() {

		return Joiner.on(" ").join(stringList());

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.model.FieldList#stringList()

	 */

	@Override

	public List<String> stringList() {

		return Lists.newArrayList(Iterables.transform(fields, new Function<Field, String>() {

			@Override

			public String apply(final Field f) {

				return f.stringValue();

			}

		}));

	}

	@Override

	public String toString() {

		return stringList().toString();

	}

}

package eu.dnetlib.pace.model;

import java.util.Iterator;

import java.util.List;

import org.apache.commons.collections.iterators.SingletonIterator;

import eu.dnetlib.pace.config.Type;

/**

 * The Class FieldValueImpl.

 */

public class FieldValueImpl extends AbstractField implements FieldValue {

	/** The value. */

	private Object value = null;

	/**

	 * Instantiates a new field value impl.

	 */

	public FieldValueImpl() {}

	/**

	 * Instantiates a new field value impl.

	 * 

	 * @param type

	 *            the type

	 * @param name

	 *            the name

	 * @param value

	 *            the value

	 */

	public FieldValueImpl(final Type type, final String name, final Object value) {

		super(type, name);

		this.value = value;

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.model.Field#isEmpty()

	 */

	@Override

	public boolean isEmpty() {

		if (value == null) return false;

		switch (type) {

		case String:

			return value.toString().isEmpty();

		case List:

			List<?> list = (List<?>) value;

			return list.isEmpty() || ((FieldValueImpl) list.get(0)).isEmpty();

		default:

			return true;

		}

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.model.FieldValue#getValue()

	 */

	@Override

	public Object getValue() {

		return value;

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.model.FieldValue#setValue(java.lang.Object)

	 */

	@Override

	public void setValue(final Object value) {

		this.value = value;

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.model.Field#stringValue()

	 */

	@Override

	// @SuppressWarnings("unchecked")

	public String stringValue() {

		return String.valueOf(getValue());

		// switch (getType()) {

		//

		// case Int:

		// return String.valueOf(getValue());

		// case List:

		// return Joiner.on(" ").join((List<String>) getValue());

		// case String:

		// return (String) getValue();

		// default:

		// throw new IllegalArgumentException("Unknown type: " + getType().toString());

		// }

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.lang.Iterable#iterator()

	 */

	@Override

	@SuppressWarnings("unchecked")

	public Iterator<Field> iterator() {

		return new SingletonIterator(this);

	}

}

import java.util.List;

import com.google.common.base.Splitter;

import com.google.common.collect.Lists;

 * The schema is composed by field definitions (FieldDef). Each field has a type, a name, and an associated distance algorithm.

	private String path;

	public FieldDef(final String name, final String path, final DistanceAlgo algo, final boolean ignoreMissing) {

		this.path = path;

	// def apply(s: String): Field[A]

	public Field apply(final Type type, final String s) {

			return new FieldValueImpl(type, name, Integer.parseInt(s));

			return new FieldValueImpl(type, name, s);

		case List:

			return new FieldListImpl(name);

		return name;

	public String getPath() {

		return path;

	}

	public List<String> getPathList() {

		return Lists.newArrayList(Splitter.on(PATH_SEPARATOR).split(getPath()));

	}

		return getName() + " { \n\talgo='" + getAlgo().getClass().getSimpleName() + "' \n\tweigth='" + getAlgo().getWeight() + "' \n\tignoreMissing='"

				+ isIgnoreMissing() + "'\n }";

/**

 * The Class MapDocumentComparator.

 */

	/** The comparator field. */

	/**

	 * Instantiates a new map document comparator.

	 * 

	 * @param comparatorField

	 *            the comparator field

	 */

	public MapDocumentComparator(final String comparatorField) {

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.Comparator#compare(java.lang.Object, java.lang.Object)

	 */

	public int compare(final Document d1, final Document d2) {

		if (d1.values(comparatorField).isEmpty() || d2.values(comparatorField).isEmpty()) return 0;

		String o1 = d1.values(comparatorField).get(0).stringValue();

		String o2 = d2.values(comparatorField).get(0).stringValue();

		if ((o1 == null) || (o2 == null)) return 0;

		String to1 = NGramUtils.cleanupForOrdering(o1);

		String to2 = NGramUtils.cleanupForOrdering(o2);

import java.lang.reflect.Type;

import com.google.gson.GsonBuilder;

import com.google.gson.InstanceCreator;

import com.google.gson.JsonDeserializationContext;

import com.google.gson.JsonDeserializer;

import com.google.gson.JsonElement;

import com.google.gson.JsonParseException;

/**

 * The Class MapDocumentSerializer.

 */

public class MapDocumentSerializer implements InstanceCreator<MapDocument> {

	@Override

	public MapDocument createInstance(final Type type) {

		return new MapDocument();

	/**

	 * Decode.

	 *

	 * @param bytes

	 *            the bytes

	 * @return the map document

	 */

	public static MapDocument decode(final byte[] bytes) {

		GsonBuilder gson = new GsonBuilder();

		gson.registerTypeAdapter(Field.class, new JsonDeserializer<Field>() {

			@Override

			public Field deserialize(final JsonElement json, final Type typeOfT, final JsonDeserializationContext context) throws JsonParseException {

				FieldListImpl fl = new FieldListImpl();

				if (json.isJsonObject()) {

					String name = json.getAsJsonObject().get("name").getAsString();

					String type = json.getAsJsonObject().get("type").getAsString();

					String value = json.getAsJsonObject().get("value").getAsString();

					fl.add(new FieldValueImpl(eu.dnetlib.pace.config.Type.valueOf(type), name, value));

				}

				return fl;

			}

		});

		return gson.create().fromJson(new String(bytes), MapDocument.class);

	}

	/**

	 * To string.

	 *

	 * @param doc

	 *            the doc

	 * @return the string

	 */

	public static String toString(final MapDocument doc) {

	/**

	 * To byte array.

	 *

	 * @param doc

	 *            the doc

	 * @return the byte[]

	 */

	public static byte[] toByteArray(final MapDocument doc) {

package eu.dnetlib.pace.model;

import eu.dnetlib.pace.config.Type;

/**

 * The Class AbstractField.

 */

public abstract class AbstractField implements Field {

	/** The type. */

	protected Type type = Type.String;

	/** The name. */

	protected String name;

	/**

	 * Instantiates a new abstract field.

	 */

	protected AbstractField() {}

	/**

	 * Instantiates a new abstract field.

	 *

	 * @param type

	 *            the type

	 * @param name

	 *            the name

	 */

	protected AbstractField(final Type type, final String name) {

		this.type = type;

		this.name = name;

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.model.Field#getName()

	 */

	@Override

	public String getName() {

		return name;

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.model.Field#getType()

	 */

	@Override

	public Type getType() {

		return type;

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.model.Field#setName(java.lang.String)

	 */

	@Override

	public void setName(final String name) {

		this.name = name;

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.pace.model.Field#setType(eu.dnetlib.pace.config.Type)

	 */

	@Override

	public void setType(final Type type) {

		this.type = type;

	}

}

/**

 * The Class MapDocument.

 */

	/** The identifier. */

	/** The field map. */

	private Map<String, FieldListImpl> fieldMap;

	/**

	 * Instantiates a new map document.

	 */

	/**

	 * Instantiates a new map document.

	 *

	 * @param identifier

	 *            the identifier

	 * @param fieldMap

	 *            the field map

	 */

	public MapDocument(final String identifier, final Map<String, FieldListImpl> fieldMap) {

	/**

	 * Instantiates a new map document.

	 *

	 * @param identifier

	 *            the identifier

	 * @param data

	 *            the data

	 */

	public MapDocument(final String identifier, final byte[] data) {

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.pace.model.document.Document#fields()

	 */

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.pace.model.document.Document#values(java.lang.String)

	 */

	public FieldList values(final String name) {

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.pace.model.document.Document#fieldNames()

	 */

	/*

	 * (non-Javadoc)

	 *

	 * @see java.lang.Object#toString()

	 */

		// return String.format("Document(%s)", fieldMap.toString());

	/**

	 * To byte array.

	 *

	 * @return the byte[]

	 */

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.pace.model.document.Document#getIdentifier()

	 */

	/**

	 * Sets the identifier.

	 *

	 * @param identifier

	 *            the new identifier

	 */

	public void setIdentifier(final String identifier) {

	/**

	 * Gets the field map.

	 *

	 * @return the field map

	 */

	public Map<String, FieldListImpl> getFieldMap() {

	/**

	 * Sets the field map.

	 *

	 * @param fieldMap

	 *            the field map

	 */

	public void setFieldMap(final Map<String, FieldListImpl> fieldMap) {

/**

 * The Interface Document. Models the common operations available on a Pace Document.

 */

	/**

	 * Gets the identifier.

	 * 

	 * @return the identifier

	 */

	/**

	 * Fields.

	 * 

	 * @return the iterable

	 */

	/**

	 * Values.

	 * 

	 * @param name

	 *            the name

	 * @return the field list

	 */

	FieldList values(String name);

	/**

	 * Field names.

	 * 

	 * @return the sets the

	 */

package eu.dnetlib.pace.model;

import java.util.List;

/**

 * The Interface FieldList.

 */

public interface FieldList extends List<Field>, Field {

	/**

	 * String list.

	 * 

	 * @return the list

	 */

	public List<String> stringList();

}

package eu.dnetlib.pace.model;

/**

 * The Interface FieldValue.

 */

public interface FieldValue extends Field {

	/**

	 * Gets the value.

	 * 

	 * @return the value

	 */

	public Object getValue();

	/**

	 * Sets the value.

	 * 

	 * @param value

	 *            the new value

	 */

	public void setValue(final Object value);

}

/**

 * The Interface Field.

 */

public interface Field extends Iterable<Field> {

	/**

	 * Gets the name.

	 * 

	 * @return the name

	 */

	public String getName();

	/**

	 * Sets the name.

	 * 

	 * @param name

	 *            the new name

	 */

	public void setName(String name);

	/**

	 * Gets the type.

	 * 

	 * @return the type

	 */

	public Type getType();

	/**

	 * Sets the type.

	 * 

	 * @param type

	 *            the new type

	 */

	public void setType(Type type);

	/**

	 * Checks if is empty.

	 * 

	 * @return true, if is empty

	 */

	public boolean isEmpty();

	/**

	 * String value.

	 * 

	 * @return the string

	 */

	public String stringValue();

/**

 * The Class DocumentBuilder.

 */

	/**

	 * New instance.

	 *

	 * @param identifier

	 *            the identifier

	 * @param fieldMap

	 *            the field map

	 * @return the map document

	 */

	public static MapDocument newInstance(final String identifier, final Map<String, FieldListImpl> fieldMap) {

	/**

	 * New instance.

	 *

	 * @param identifier

	 *            the identifier

	 * @param fieldMap

	 *            the field map

	 * @return the map document

	 */

import java.util.Set;

import com.google.common.collect.Sets;

import eu.dnetlib.pace.condition.DoiExactMatch;

import eu.dnetlib.pace.condition.ExactMatch;

import eu.dnetlib.pace.condition.SizeMatch;

import eu.dnetlib.pace.distance.SortedJaroWinkler;

import eu.dnetlib.pace.distance.SortedLevel2JaroWinkler;

	public List<CondDef> strictConditions() {

		return parseConds("strict");

	}

	@Override

		return new FieldDef(identifierField(), null, new NullDistanceAlgo(), false);

				final String path = config.getString(String.format("pace.conf.model%s.%s.path", base, name));

				Boolean ignoreMissing = config.getBoolean(String.format("pace.conf.model%s.%s.ignoreMissing", base, name));

				return new FieldDef(name, path, getAlgo(base, name, weight), ignoreMissing);

				case SortedJaroWinkler:

					return new SortedJaroWinkler(w);

				case SortedLevel2JaroWinkler:

					return new SortedLevel2JaroWinkler(w);

		final Map<String, ?> modelMap = (Map<String, ?>) config.getObject(String.format("pace.conf.%sconditions", base));

				final List<String> fieldList = config.getList(String.format("pace.conf.%sconditions.%s.fields", base, e.getKey()));

				final Set<String> fieldSet = Sets.newHashSet(fieldList);

				final List<FieldDef> fields = Lists.newArrayList(Iterables.filter(fields(), new Predicate<FieldDef>() {

					@Override

					public boolean apply(final FieldDef fd) {

						return fieldSet.contains(fd.getName());

					}

				}));

			private ConditionAlgo getCondAlgo(final List<FieldDef> fields, final Cond condName) {

				case sizeMatch:

					return new SizeMatch(fields);

				case exactMatch:

					return new ExactMatch(fields);

				case doiExactMatch:

					return new DoiExactMatch(fields);

				(Map<String, Integer>) config.getObject(String.format("pace.conf.clustering%s.%s.params", base, e.getKey()));

/**

 * Enumerates the distance Algos.

 */

	/** The Jaro winkler. */

	JaroWinkler,

	/** The Jaro winkler title. */

	JaroWinklerTitle,

	/** The Levenstein. */

	Levenstein,

	/** The Level2 jaro winkler. */

	Level2JaroWinkler,

	/** The Level2 levenstein. */

	Level2Levenstein,

	/** The Sub string levenstein. */

	SubStringLevenstein,

	/** The Year levenstein. */

	YearLevenstein,

	/** The Sorted jaro winkler. */

	SortedJaroWinkler,

	/** The Sorted level2 jaro winkler. */

	SortedLevel2JaroWinkler,

	/** The Null. */

	Null

/**

 * The Enum Cond.

 */

	/** The year match. */

	yearMatch,

	/** The title version match. */

	titleVersionMatch,

	/** The size match. */

	sizeMatch,