D-Net

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-openaireplus-mapping-utils</artifactId>

	<packaging>jar</packaging>

	<version>7.0.1</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-7.0.1</developerConnection>

	</scm>

	<properties>

		<skipITs>true</skipITs>

	</properties>

	<build>

		<plugins>

			<plugin>

				<groupId>org.apache.maven.plugins</groupId>

				<artifactId>maven-failsafe-plugin</artifactId>

				<version>2.19.1</version>

				<executions>

					<execution>

						<id>integration-test</id>

						<goals>

							<goal>integration-test</goal>

						</goals>

					</execution>

					<execution>

						<id>verify</id>

						<goals>

							<goal>verify</goal>

						</goals>

					</execution>

				</executions>

				<configuration>

					<skipITs>${skipITs}</skipITs>

				</configuration>

			</plugin>

		</plugins>

	</build>

	<dependencies>

		<dependency>

			<groupId>com.google.guava</groupId>

			<artifactId>guava</artifactId>

			<version>${google.guava.version}</version>

		</dependency>		

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>com.ximpleware</groupId>

			<artifactId>vtd-xml</artifactId>

			<version>[2.12, 3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>commons-codec</groupId>

			<artifactId>commons-codec</artifactId>

			<version>${commons.codec.version}</version>

		</dependency>

		<dependency>

			<groupId>dom4j</groupId>

			<artifactId>dom4j</artifactId>

			<version>${dom4j.version}</version>

			<exclusions>

				<exclusion>

					<artifactId>xml-apis</artifactId>

					<groupId>xml-apis</groupId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>net.sf.supercsv</groupId>

			<artifactId>super-csv</artifactId>

			<version>2.4.0</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-openaire-data-protos</artifactId>

			<version>[3.9.8]</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-pace-core</artifactId>

			<version>[3.0.0,4.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-misc-utils</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-hadoop-commons</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-index-solr-common</artifactId>

			<version>[3.0.1,4.0.0)</version>

		</dependency>	

		<dependency>

			<groupId>com.googlecode.protobuf-java-format</groupId>

			<artifactId>protobuf-java-format</artifactId>

			<version>1.2</version>

		</dependency>

		<dependency>

			<groupId>org.apache.commons</groupId>

			<artifactId>commons-lang3</artifactId>

			<version>3.5</version>

		</dependency>

		<!-- test deps -->

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-openaireplus-profiles</artifactId>

			<version>[1.0.0,2.0.0)</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.mongodb</groupId>

			<artifactId>mongo-java-driver</artifactId>

			<version>${mongodb.driver.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.springframework</groupId>

			<artifactId>spring-context</artifactId>

			<version>${spring.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.springframework</groupId>

			<artifactId>spring-core</artifactId>

			<version>${spring.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.springframework</groupId>

			<artifactId>spring-test</artifactId>

			<version>${spring.version}</version>

			<scope>test</scope>

		</dependency>

	</dependencies>

</project>

package eu.dnetlib.data.bulktag;

import com.google.gson.Gson;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import java.util.ArrayList;

import java.util.List;

/**

 * Created by miriam on 01/08/2018.

 */

public class Community {

    private static final Log log = LogFactory.getLog(Community.class);

    private String id;

    private List<String> subjects = new ArrayList<>();

    private List<Datasource> datasources = new ArrayList<>();

    private List<ZenodoCommunity> zenodoCommunities = new ArrayList<>();

    private List<Organization> organizationCommunity = new ArrayList<>();

    public List<Organization> getOrganizationCommunity() {

        return organizationCommunity;

    }

    public void setOrganizationCommunity(List<Organization> organizationCommunity) {

        this.organizationCommunity = organizationCommunity;

    }

    public String toJson() {

        final Gson g = new Gson();

        return g.toJson(this);

    }

    public boolean isValid() {

        return !getSubjects().isEmpty() || !getDatasources().isEmpty() || !getZenodoCommunities().isEmpty();

    }

    public String getId() {

        return id;

    }

    public void setId(String id) {

        this.id = id;

    }

    public List<String> getSubjects() {

        return subjects;

    }

    public void setSubjects(List<String> subjects) {

        this.subjects = subjects;

    }

    public List<Datasource> getDatasources() {

        return datasources;

    }

    public void setDatasources(List<Datasource> datasources) {

        this.datasources = datasources;

    }

    public List<ZenodoCommunity> getZenodoCommunities() {

        return zenodoCommunities;

    }

    public void setZenodoCommunities(List<ZenodoCommunity> zenodoCommunities) {

        this.zenodoCommunities = zenodoCommunities;

    }

}

package eu.dnetlib.data.index;

import eu.dnetlib.functionality.index.solr.feed.StreamingInputDocumentFactory;

import eu.dnetlib.miscutils.datetime.HumanTime;

import eu.dnetlib.miscutils.functional.UnaryFunction;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.SolrQuery;

import org.apache.solr.client.solrj.SolrServerException;

import org.apache.solr.client.solrj.impl.CloudSolrClient;

import org.apache.solr.client.solrj.response.QueryResponse;

import org.apache.solr.client.solrj.response.UpdateResponse;

import org.apache.solr.common.SolrInputDocument;

import java.io.Closeable;

import java.io.IOException;

import java.text.SimpleDateFormat;

import java.util.Date;

import java.util.List;

/**

 * Created by michele on 11/11/15.

 */

public class CloudIndexClient implements Closeable {

	private static final Log log = LogFactory.getLog(CloudIndexClient.class);

	private static final String INDEX_RECORD_RESULT_FIELD = "dnetResult";

	private final CloudSolrClient solrClient;

	protected CloudIndexClient(final CloudSolrClient solrServer) {

		this.solrClient = solrServer;

	}

	public int feed(final String record, final UnaryFunction<String, String> toIndexRecord) throws CloudIndexClientException {

		return feed(record, toIndexRecord, true);

	}

	public int feed(final String record, final UnaryFunction<String, String> toIndexRecord, final boolean commit)

			throws CloudIndexClientException {

		try {

			final SolrInputDocument doc = prepareSolrDocument(record, toIndexRecord);

			if ((doc == null) || doc.isEmpty()) throw new CloudIndexClientException("Invalid solr document");

			return feed(doc, commit);

		} catch (final Throwable e) {

			throw new CloudIndexClientException("Error feeding document", e);

		}

	}

	public int feed(final SolrInputDocument document) throws CloudIndexClientException {

		return feed(document, true);

	}

	public int feed(final SolrInputDocument document, final boolean commit) throws CloudIndexClientException {

		try {

			final UpdateResponse res = solrClient.add(document);

			log.debug("feed time for single records, elapsed time: " + HumanTime.exactly(res.getElapsedTime()));

			if (res.getStatus() != 0) { throw new CloudIndexClientException("bad status: " + res.getStatus()); }

			if (commit) {

				solrClient.commit();

			}

			return res.getStatus();

		} catch (final Throwable e) {

			throw new CloudIndexClientException("Error feeding document", e);

		}

	}

	public void feed(final List<SolrInputDocument> docs, final AfterFeedingCallback callback) throws CloudIndexClientException {

		feed(docs, callback, true);

	}

	public void feed(final List<SolrInputDocument> docs, final AfterFeedingCallback callback, final boolean commit) throws CloudIndexClientException {

		try {

			if (docs.isEmpty()) {

				log.debug("Empty list of documents. Calling callback, if needed.");

				if (callback != null) {

					callback.doAfterFeeding(null);

				}

				return;

			}

			final UpdateResponse res = solrClient.add(docs);

			log.debug("feed time for " + docs.size() + " records, elapsed tipe: : " + HumanTime.exactly(res.getElapsedTime()));

			if (commit) {

				solrClient.commit();

			}

			if (callback != null) {

				callback.doAfterFeeding(res);

			}

			if (res.getStatus() != 0) throw new CloudIndexClientException("bad status: " + res.getStatus());

		} catch (final Throwable e) {

			throw new CloudIndexClientException("Error feeding documents", e);

		}

	}

	public SolrInputDocument prepareSolrDocument(final String record, final UnaryFunction<String, String> toIndexRecord)

			throws CloudIndexClientException {

		try {

			final StreamingInputDocumentFactory documentFactory = new StreamingInputDocumentFactory();

			final String version = (new SimpleDateFormat("yyyy-MM-dd\'T\'hh:mm:ss\'Z\'")).format(new Date());

			final String indexRecord = toIndexRecord.evaluate(record);

			if (log.isDebugEnabled()) {

				log.debug("***************************************\nSubmitting index record:\n" + indexRecord + "\n***************************************\n");

			}

			return documentFactory.parseDocument(version, indexRecord, INDEX_RECORD_RESULT_FIELD);

		} catch (final Throwable e) {

			throw new CloudIndexClientException("Error creating solr document", e);

		}

	}

	public boolean isRecordIndexed(final String id) throws CloudIndexClientException {

		final QueryResponse res = query("objidentifier:\"" + id + "\"", null);

		return res.getResults().size() > 0;

	}

	public int remove(final String id) throws CloudIndexClientException {

		return remove(id, true);

	}

	public int remove(final String id, final boolean commit) throws CloudIndexClientException {

		String q = String.format("objidentifier:\"%s\" OR resultdupid:\"%s\"", id, id);

		try {

			final UpdateResponse res = solrClient.deleteByQuery(q);

			if (commit) {

				solrClient.commit();

			}

			return res.getResponse().size();

		} catch (final Throwable e) {

			throw new CloudIndexClientException("Error removing documents", e);

		}

	}

	public int count(final String query) throws CloudIndexClientException {

		final QueryResponse res = query(query, 0);

		return res.getResults().size();

	}

	public QueryResponse query(final String query, Integer rows) throws CloudIndexClientException {

		try {

			final SolrQuery solrQuery = new SolrQuery();

			solrQuery.setQuery(query);

			if(rows != null && rows >= 0) {

				solrQuery.setRows(rows);

			}

			return solrClient.query(solrQuery);

		} catch (final Throwable e) {

			throw new CloudIndexClientException("Error searching documents", e);

		}

	}

	public void close() throws IOException {

		if (solrClient != null) {

			solrClient.close();

		}

	}

	public void commit() throws CloudIndexClientException {

		if(solrClient != null) {

			try {

				solrClient.commit();

			} catch (SolrServerException | IOException e) {

				throw new CloudIndexClientException(e.getMessage());

			}

		}

	}

	public interface AfterFeedingCallback {

		void doAfterFeeding(final UpdateResponse response);

	}

}

package eu.dnetlib.data.index;

import eu.dnetlib.functionality.index.utils.ZkServers;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.impl.CloudSolrClient;

import org.apache.solr.client.solrj.response.SolrPingResponse;

/**

 * Created by michele on 11/11/15.

 */

public class CloudIndexClientFactory {

	private static final Log log = LogFactory.getLog(CloudIndexClientFactory.class);

	public static CloudIndexClient newIndexClient(final String baseURL, final String collection, final boolean parallelUpdates)

			throws CloudIndexClientException {

		try {

			log.info(String.format("Initializing solr server (%s) ...", baseURL));

			final ZkServers zk = ZkServers.newInstance(baseURL);

			final CloudSolrClient client = new CloudSolrClient.Builder(zk.getHosts(), zk.getChroot())

					.withParallelUpdates(parallelUpdates)

					.build();

			client.connect();

			client.setDefaultCollection(collection);

			final SolrPingResponse rsp = client.ping();

			if (rsp.getStatus() != 0) {

				log.error("Invalid connection to solr Server (status = 0)");

				throw new CloudIndexClientException("Invalid connection to solr Server (status = 0)");

			}

			return new CloudIndexClient(client);

		} catch (Throwable e) {

			log.error("The initialization of indexClient is FAILED", e);

			throw new CloudIndexClientException("The initialization of indexClient is FAILED", e);

		}

	}

}

package eu.dnetlib.data.transform;

import com.wcohen.ss.JaroWinkler;

import eu.dnetlib.data.bulktag.Pair;

import eu.dnetlib.data.proto.FieldTypeProtos.Author;

import eu.dnetlib.data.proto.FieldTypeProtos.KeyValue;

import eu.dnetlib.pace.model.Person;

import org.apache.commons.lang3.StringUtils;

import java.text.Normalizer;

import java.util.*;

import java.util.function.Function;

import static java.util.stream.Collectors.*;

public class AuthorMerger {

    private static final Double THRESHOLD = 0.95;

    private static final String ORCID = "orcid";

    private static final int MAX_AUTHORS = 200;

    public static List<Author> merge(final Collection<List<Author>> authors, final double threshold) {

        return merge(authors, THRESHOLD);

    }

    public static List<Author> merge(final Collection<List<Author>> authors) {

        return doMerge(

                authors.stream()

                        .map(group -> group.stream()

                                .map(AuthorMerger::fixORCID)

                                .collect(toList()))

                        .collect(toList()));

    }

    private static List<Author> doMerge(final Collection<List<Author>> authors) {

        final List<Author> res = new ArrayList<>();

        if (authors.isEmpty()) {

            return res;

        }

        if (authors.size() == 1) {

            return authors.iterator().next();

        }

        final TreeMap<Integer, List<List<Author>>> byOrcidCount = new TreeMap<>(

                authors.stream()

                        .collect(groupingBy(AuthorMerger::countOrcid))

                        .entrySet().stream()

                        .filter(e -> e.getKey() > 0)

                        .collect(toMap(

                                Map.Entry::getKey,

                                Map.Entry::getValue

                        )));

        if (byOrcidCount == null || byOrcidCount.isEmpty()) {

            return authors.iterator().next();

        }

        final Map.Entry<Integer, List<List<Author>>> mostOrcid = byOrcidCount.lastEntry();

        if (mostOrcid.getKey() > 0) {

            final List<Author> pivots = mostOrcid.getValue().iterator().next();

            res.addAll(mostOrcid.getValue().iterator().next().stream()

                    .filter(a -> hasOrcid(a))

                    .collect(toList()));

            if (pivots.size() == res.size()) {

                return res;

            }

            final Collection<Author> authorList = authors.stream()

                    .filter(g -> !g.equals(pivots))

                    .flatMap(List::stream)

                    .filter(a -> hasOrcid(a))

                    .limit(MAX_AUTHORS)

                    .map(a -> {

                        final String orcid = a.getPidList().stream()

                                .filter(p -> p.getKey().equalsIgnoreCase(ORCID))

                                .findFirst()

                                .get().getValue();

                        return new Pair<String, Author>(orcid, a);

                    })

                    .collect(toMap(

                            p -> p.getFst(),

                            p -> p.getSnd(),

                            (p1, p2) -> p2))

                    .values();

            pivots.stream().filter(a -> !hasOrcid(a)).forEach(pivot -> {

                final Author.Builder b = Author.newBuilder(pivot);

                authorList.parallelStream()

                        .map(a -> {

                            return new Pair<Double, Author>(sim(a, pivot), a);

                        })

                        .filter(p -> p.getFst() >= THRESHOLD)

                        .forEach(p -> {

                            b.mergeFrom(p.getSnd());

                        });

                Collection<KeyValue> pids = b.getPidList().stream()

                        .collect(toMap(

                                kv -> kv.getKey(),

                                Function.identity(),

                                (kv1, kv2) -> kv2

                        )).values();

                b.clearPid();

                b.addAllPid(pids);

                res.add(b.build());

            });

        }

        return res;

    }

    private static Author fixORCID(final Author author) {

        final Author.Builder b = Author.newBuilder(author);

        for(KeyValue.Builder pid : b.getPidBuilderList()) {

            if (pid.getKey().toLowerCase().contains(ORCID)) {

                pid.setKey("ORCID");

                if (pid.getValue().contains("orcid.org")) {

                    pid.setValue(StringUtils.substringAfterLast(pid.getValue(), "/"));

                }

            }

        }

        return b.build();

    }

    private static int countOrcid(final List<Author> authors) {

        return authors.stream()

                .map(a -> {

                    return hasOrcid(a) ? 1 : 0;

                })

                .mapToInt(Integer::intValue)

                .sum();

    }

    private static boolean hasOrcid(Author a) {

        return a.getPidList().stream().anyMatch(p -> p.getKey().equalsIgnoreCase(ORCID));

    }

    private static Double sim(Author a, Author b) {

        final Person pa = parse(a);

        final Person pb = parse(b);

        if (pa.isAccurate() & pb.isAccurate()) {

            return new JaroWinkler().score(

                    normalize(pa.getSurnameString()),

                    normalize(pb.getSurnameString()));

        } else {

            return new JaroWinkler().score(

                        normalize(pa.getNormalisedFullname()),

                        normalize(pb.getNormalisedFullname()));

        }

    }

    private static Person parse(Author author) {

        if (author.hasSurname()) {

            return new Person(author.getSurname() + ", " + author.getName(), false);

        } else {

            return new Person(author.getFullname(), false);

        }

    }

    private static String normalize(final String s) {

        return nfd(s).toLowerCase()

                // do not compact the regexes in a single expression, would cause StackOverflowError in case of large input strings

                .replaceAll("(\\W)+", " ")

                .replaceAll("(\\p{InCombiningDiacriticalMarks})+", " ")

                .replaceAll("(\\p{Punct})+", " ")

                .replaceAll("(\\d)+", " ")

                .replaceAll("(\\n)+", " ")

                .trim();

    }

    private static String nfd(final String s) {

        return Normalizer.normalize(s, Normalizer.Form.NFD);

    }

}

package eu.dnetlib.data.index;

/**

 * Created by michele on 23/11/15.

 */

public class CloudIndexClientException extends Exception {

	public CloudIndexClientException(final String message) {

		super(message);

	}

	public CloudIndexClientException(final String message, final Throwable cause) {

		super(message, cause);

	}

}

package eu.dnetlib.data.transform;

import eu.dnetlib.data.proto.FieldTypeProtos;

import org.apache.commons.lang.StringUtils;

import java.time.Year;

import java.util.*;

import java.util.stream.Collectors;

import static java.util.Collections.reverseOrder;

import static java.util.Map.Entry.comparingByValue;

import static java.util.stream.Collectors.toMap;

import static org.apache.commons.lang.StringUtils.endsWith;

import static org.apache.commons.lang.StringUtils.substringBefore;

public class DatePicker {

    private static final String DATE_PATTERN = "\\d{4}-\\d{2}-\\d{2}";

    private static final String DATE_DEFAULT_SUFFIX = "01-01";

    private static final int YEAR_LB = 1300;

    private static final int YEAR_UB = Year.now().getValue() + 5;

    public static FieldTypeProtos.StringField pick(final Collection<String> dateofacceptance) {

        final Map<String, Integer> frequencies = dateofacceptance

                .parallelStream()

                .filter(StringUtils::isNotBlank)

                .collect(

                        Collectors.toConcurrentMap(

                                w -> w, w -> 1, Integer::sum));

        if (frequencies.isEmpty()) {

            return FieldTypeProtos.StringField.newBuilder().setValue("").build();

        }

        final FieldTypeProtos.StringField.Builder date = FieldTypeProtos.StringField.newBuilder().setValue(frequencies.keySet().iterator().next());

        // let's sort this map by values first, filtering out invalid dates

        final Map<String, Integer> sorted = frequencies

                .entrySet()

                .stream()

                .filter(d -> StringUtils.isNotBlank(d.getKey()))

                .filter(d -> d.getKey().matches(DATE_PATTERN))

                .filter(d -> inRange(d.getKey()))

                .sorted(reverseOrder(comparingByValue()))

                .collect(

                        toMap(

                                Map.Entry::getKey,

                                Map.Entry::getValue, (e1, e2) -> e2,

                                LinkedHashMap::new));

        // shortcut

        if (sorted.size() == 0) {

            return date.build();

        }

        // voting method (1/3 + 1) wins

        if (sorted.size() >= 3) {

            final int acceptThreshold = (sorted.size() / 3) + 1;

            final List<String> accepted = sorted.entrySet().stream()

                    .filter(e -> e.getValue() >= acceptThreshold)

                    .map(e -> e.getKey())

                    .collect(Collectors.toList());

            // cannot find strong majority

            if (accepted.isEmpty()) {

                final int max = sorted.values().iterator().next();

                Optional<String> first = sorted.entrySet().stream()

                        .filter(e -> e.getValue() == max && !endsWith(e.getKey(), DATE_DEFAULT_SUFFIX))

                        .map(Map.Entry::getKey)

                        .findFirst();

                if (first.isPresent()) {

                    return date.setValue(first.get()).build();

                }

                return date.setValue(sorted.keySet().iterator().next()).build();

            }

            if (accepted.size() == 1) {

                return date.setValue(accepted.get(0)).build();

            } else {

                final Optional<String> first = accepted.stream()

                        .filter(d -> !endsWith(d, DATE_DEFAULT_SUFFIX))

                        .findFirst();

                if (first.isPresent()) {

                    return date.setValue(first.get()).build();

                }

                return date.build();

            }

            //1st non YYYY-01-01 is returned

        } else {

            if (sorted.size() == 2) {

                for (Map.Entry<String, Integer> e : sorted.entrySet()) {

                    if (!endsWith(e.getKey(), DATE_DEFAULT_SUFFIX)) {

                        return date.setValue(e.getKey()).build();

                    }

                }

            }

            // none of the dates seems good enough, return the 1st one

            return date.setValue(sorted.keySet().iterator().next()).build();

        }

    }

    private static boolean inRange(final String date) {

        final int year = Integer.parseInt(substringBefore(date, "-"));

        return year >= YEAR_LB && year <= YEAR_UB;

    }

}

package eu.dnetlib.data.transform;

import com.google.common.collect.ImmutableList;

import com.google.common.collect.Ordering;

import eu.dnetlib.data.proto.OafProtos.Oaf;

import eu.dnetlib.data.proto.SpecialTrustProtos.SpecialTrust;

import org.apache.commons.lang3.StringUtils;

public class TrustOrdering extends Ordering<Oaf> {

	@Override

	public int compare(Oaf left, Oaf right) {

		String lTrust = left.getDataInfo().getTrust();

		String rTrust = right.getDataInfo().getTrust();

		if (lTrust.equals(rTrust)) return 0;

		if (lTrust.equals(SpecialTrust.INFINITE.toString())) return 1;

		if (rTrust.equals(SpecialTrust.INFINITE.toString())) return -1;

		if (lTrust.equals(SpecialTrust.NEUTRAL.toString())) return 1;

		if (rTrust.equals(SpecialTrust.NEUTRAL.toString())) return -1;

		return Float.compare(

				Float.parseFloat(StringUtils.isBlank(lTrust) ? "0.9" : lTrust),

				Float.parseFloat(StringUtils.isBlank(rTrust) ? "0.9" : rTrust));

	}

	public static ImmutableList<Oaf> sort(Iterable<Oaf> entities) {

		return new TrustOrdering().immutableSortedCopy(entities);

	}

}

package eu.dnetlib.data.transform;

import java.util.List;

import java.util.function.Function;

import com.google.common.collect.Lists;

import eu.dnetlib.data.mapreduce.util.OafDecoder;

import eu.dnetlib.data.mapreduce.util.OafEntityDecoder;

import eu.dnetlib.data.proto.OafProtos.Oaf;

public class OafToRowMapper implements Function<Oaf, List<Row>> {

	public static final String BODY = "body";

	@Override

	public List<Row> apply(final Oaf oaf) {

		final List<Row> rows = Lists.newArrayList();

		final OafDecoder d = OafDecoder.decode(oaf);

		final OafEntityDecoder entity = d.decodeEntity();

		final Row r = new Row(d.getCFQ(), entity.getId());

		switch (entity.getType()) {

		case project:

			r.addColumn(new Column<>(BODY, oaf.toByteArray()));

			break;

		case result:

			oaf.getEntity().getCachedOafRelList().stream()

					.map(cachedRel -> {

						final Oaf.Builder oafRel = Oaf.newBuilder(cachedRel);

						oafRel.getRelBuilder().clearCachedOafTarget();

						return oafRel.build();

					}).forEach(oafRel -> r.addColumn(new Column<>(OafDecoder.decode(oafRel).getCFQ(), oafRel.toByteArray())));

			break;

		case datasource:

			break;

		case organization:

			break;

		}

		return rows;

	}

}

package eu.dnetlib.data.bulktag;

import com.google.gson.Gson;

import eu.dnetlib.data.bulktag.selectioncriteria.VerbResolver;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Node;

/**

 * Created by miriam on 01/08/2018.

 */

public class Datasource {

    private static final Log log = LogFactory.getLog(Datasource.class);

    private String openaireId;

    private SelectionConstraints selectionConstraints;

    public SelectionConstraints getSelCriteria() {

        return selectionConstraints;

    }

    public SelectionConstraints getSelectionConstraints() {

        return selectionConstraints;

    }

    public void setSelectionConstraints(SelectionConstraints selectionConstraints) {

        this.selectionConstraints = selectionConstraints;

    }

    public void setSelCriteria(SelectionConstraints selCriteria) {

        this.selectionConstraints = selCriteria;

    }

    public String getOpenaireId() {

        return openaireId;

    }

    public void setOpenaireId(String openaireId) {

        this.openaireId = openaireId;

    }

    private void setSelCriteria(String json, VerbResolver resolver){

        log.info("Selection constraints for datasource = " + json);

        selectionConstraints = new Gson().fromJson(json, SelectionConstraints.class);

        selectionConstraints.setSelection(resolver);

    }

    public void setSelCriteria(Node n, VerbResolver resolver){

        try{

            setSelCriteria(n.getText(),resolver);

        }catch(Exception e) {

            log.info("not set selection criteria... ");

            selectionConstraints =null;

        }

    }

}

package eu.dnetlib.data.bulktag;

import com.google.gson.Gson;

import eu.dnetlib.data.bulktag.selectioncriteria.VerbResolver;

import org.dom4j.Node;

public class Organization {

    private String organizationId;

    private SelectionConstraints selCriteria;

    public String getOrganizationId() {

        return organizationId;

    }

    public void setOrganizationId(String organizationId) {

        this.organizationId = organizationId;

    }

    public SelectionConstraints getSelCriteria() {

        return selCriteria;

    }

    public void setSelCriteria(SelectionConstraints selCriteria) {

        this.selCriteria = selCriteria;

    }

    private void setSelCriteria(String json){

        //Type collectionType = new TypeToken<Collection<Constraints>>(){}.getType();

        selCriteria = new Gson().fromJson(json, SelectionConstraints.class);

    }

    public void setSelCriteria(Node n){

        if (n==null){

            selCriteria = null;

        }else{

            setSelCriteria(n.getText());

        }

    }

}

package eu.dnetlib.data.bulktag;

import com.google.common.reflect.TypeToken;

import com.google.gson.Gson;

import eu.dnetlib.data.bulktag.selectioncriteria.VerbResolver;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import java.io.Serializable;

import java.lang.reflect.InvocationTargetException;

import java.lang.reflect.Type;

import java.util.Collection;

import java.util.List;

import java.util.Map;

/**

 * Created by miriam on 02/08/2018.

 */

public class Constraints implements Serializable {

    private static final Log log = LogFactory.getLog(Constraints.class);

    //private ConstraintEncapsulator ce;

    private List<Constraint> constraint;

    public Constraints() {

    }

    public List<Constraint> getConstraint() {

        return constraint;

    }

    public void setConstraint(List<Constraint> constraint) {

        this.constraint = constraint;

    }

    public void setSc(String json){

        Type collectionType = new TypeToken<Collection<Constraint>>(){}.getType();

        constraint = new Gson().fromJson(json, collectionType);

    }

    void setSelection(VerbResolver resolver) {

        for(Constraint st: constraint){

            try {

                st.setSelection(resolver);

            } catch (NoSuchMethodException e) {

                log.error(e.getMessage());

            } catch (IllegalAccessException e) {

                log.error(e.getMessage());

            } catch (InvocationTargetException e) {

                log.error(e.getMessage());

            } catch (InstantiationException e) {

                log.error(e.getMessage());

            }

        }

    }

    //Constraint in and

    public boolean verifyCriteria(final Map<String, List<String>> param) {

        for(Constraint sc : constraint) {

            boolean verified = false;

            for(String value : param.get(sc.getField())){

                if (sc.verifyCriteria(value.trim())){

                    verified = true;

                }

            }

            if(!verified)

                return verified;

        }

        return true;

    }

}

package eu.dnetlib.data.bulktag.selectioncriteria;

@VerbClass("not_equals")

public class NotEqualVerb implements Selection {

    private String param;

    public NotEqualVerb(final String param) {

        this.param = param;

    }

    public NotEqualVerb() {

    }

    public String getParam() {

        return param;

    }

    public void setParam(String param) {

        this.param = param;

    }

    @Override

    public boolean apply(String value) {

        return !value.equalsIgnoreCase(param);

    }

}

package eu.dnetlib.data.bulktag.selectioncriteria;

import org.reflections.Reflections;

import java.io.Serializable;

import java.lang.reflect.InvocationTargetException;

import java.util.Map;

import java.util.stream.Collectors;

public class VerbResolver implements Serializable {

    private final Map<String, Class<Selection>> map;

    public VerbResolver(){

        this.map = new Reflections("eu.dnetlib").getTypesAnnotatedWith(VerbClass.class).stream()

                .collect(Collectors.toMap(v -> v.getAnnotation(VerbClass.class).value(), v->(Class<Selection>)v));

    }

    public Selection getSelectionCriteria(String name, String param) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {

        return map.get(name).getDeclaredConstructor((String.class)).newInstance(param);

    }

}

package eu.dnetlib.data.bulktag;

import eu.dnetlib.data.bulktag.selectioncriteria.Selection;

import eu.dnetlib.data.bulktag.selectioncriteria.VerbResolver;

import org.springframework.beans.factory.annotation.Autowired;

import java.io.Serializable;

import java.lang.reflect.InvocationTargetException;

public class Constraint implements Serializable {

    private String verb;

    private String field;

    private String value;

    private Selection selection;

    public Constraint() {

    }

    public String getVerb() {

        return verb;

    }

    public void setVerb(String verb) {

        this.verb = verb;

    }

    public String getField() {

        return field;

    }

    public void setField(String field) {

        this.field = field;

    }

    public String getValue() {

        return value;

    }

    public void setValue(String value) {

        this.value = value;

    }

    public void setSelection(Selection sel){

        selection = sel;

    }

    public void setSelection(VerbResolver resolver) throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {

        selection = resolver.getSelectionCriteria(verb,value);

    }

    public boolean verifyCriteria(String metadata){

        return selection.apply(metadata);

    }

}

package eu.dnetlib.data.mapreduce.util;

import eu.dnetlib.data.proto.RelTypeProtos.RelType;

import eu.dnetlib.data.proto.RelTypeProtos.SubRelType;

public class RelDescriptor {

	public static final String SEPARATOR = "_";

	private final String it;

	// relType also corresponds to the Ontology code

	private final RelType relType;

	private final SubRelType subRelType;

	private final String relClass;

	// <TERM code="hasAmongTopNSimilarDocuments" encoding="resultResult_similarity_hasAmongTopNSimilarDocuments"

	//final String rd = oafRel.getRelType().toString() + "_" + oafRel.getSubRelType() + "_" + relClasses.getInverse(oafRel.getRelClass());

	//<ONTOLOGY_NAME code="dnet:result_result_relations">

	public RelDescriptor(final String value) {

		super();

		this.it = value;

		String[] s = value.split(SEPARATOR);

		this.relType = RelType.valueOf(s[0]);

		this.subRelType = SubRelType.valueOf(s[1]);

		this.relClass = s[2];

	}

	public SubRelType getSubRelType() {

		return subRelType;

	}

	public RelType getRelType() {

		return relType;

	}

	public String getRelClass() {

		return relClass;

	}

	public String getIt() {

		return it;

	}

	@Override

	public String toString() {

		return getIt();

	}

	@Override

	public int hashCode() {

		final int prime = 31;

		int result = 1;

		result = (prime * result) + ((it == null) ? 0 : it.hashCode());

		return result;

	}

	@Override

	public boolean equals(final Object obj) {

		if (this == obj) return true;

		if (obj == null) return false;

		if (getClass() != obj.getClass()) return false;

		RelDescriptor other = (RelDescriptor) obj;

		if (it == null) {

			if (other.it != null) return false;

		} else if (!it.equals(other.it)) return false;

		return true;

	}

}

package eu.dnetlib.data.transform.xml;

import java.nio.charset.Charset;

import java.security.MessageDigest;

import java.util.Collection;

import java.util.List;

import java.util.Map;

import java.util.Objects;

import java.util.Set;

import java.util.function.Function;

import java.util.stream.Collectors;

import org.apache.commons.codec.binary.Base64;

import org.apache.commons.codec.binary.Hex;

import org.apache.commons.lang.math.NumberUtils;

import org.apache.commons.lang3.StringUtils;

import org.w3c.dom.NamedNodeMap;

import org.w3c.dom.Node;

import org.w3c.dom.NodeList;

import com.google.common.base.Predicate;

import com.google.common.base.Splitter;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import com.google.common.collect.Sets;

import com.google.gson.JsonObject;

import com.google.protobuf.Descriptors.Descriptor;

import com.google.protobuf.Descriptors.FieldDescriptor;

import com.google.protobuf.InvalidProtocolBufferException;

import com.google.protobuf.Message;

import com.google.protobuf.Message.Builder;

import com.google.protobuf.ProtocolMessageEnum;

import eu.dnetlib.data.proto.DatasourceOrganizationProtos.DatasourceOrganization;

import eu.dnetlib.data.proto.DatasourceOrganizationProtos.DatasourceOrganization.Provision;

import eu.dnetlib.data.proto.DedupProtos.Dedup;

import eu.dnetlib.data.proto.DedupSimilarityProtos.DedupSimilarity;

import eu.dnetlib.data.proto.FieldTypeProtos.BoolField;

import eu.dnetlib.data.proto.FieldTypeProtos.DataInfo;

import eu.dnetlib.data.proto.FieldTypeProtos.IntField;

import eu.dnetlib.data.proto.FieldTypeProtos.Journal;

import eu.dnetlib.data.proto.FieldTypeProtos.KeyValue;

import eu.dnetlib.data.proto.FieldTypeProtos.OAIProvenance;

import eu.dnetlib.data.proto.FieldTypeProtos.OAIProvenance.OriginDescription;

import eu.dnetlib.data.proto.FieldTypeProtos.Qualifier;

import eu.dnetlib.data.proto.FieldTypeProtos.StringField;

import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty;

import eu.dnetlib.data.proto.KindProtos.Kind;

import eu.dnetlib.data.proto.OafProtos.Oaf;

import eu.dnetlib.data.proto.OafProtos.OafEntity;

import eu.dnetlib.data.proto.OafProtos.OafRel;

import eu.dnetlib.data.proto.OrganizationOrganizationProtos.OrganizationOrganization;

import eu.dnetlib.data.proto.ProjectOrganizationProtos.ProjectOrganization;

import eu.dnetlib.data.proto.ProjectOrganizationProtos.ProjectOrganization.Participation;

import eu.dnetlib.data.proto.RelMetadataProtos.RelMetadata;

import eu.dnetlib.data.proto.RelTypeProtos.RelType;

import eu.dnetlib.data.proto.RelTypeProtos.SubRelType;

import eu.dnetlib.data.proto.ResultOrganizationProtos.ResultOrganization;

import eu.dnetlib.data.proto.ResultOrganizationProtos.ResultOrganization.Affiliation;

import eu.dnetlib.data.proto.ResultProjectProtos.ResultProject;

import eu.dnetlib.data.proto.ResultProjectProtos.ResultProject.Outcome;

import eu.dnetlib.data.proto.ResultProtos.Result.Metadata;

import eu.dnetlib.data.proto.ResultResultProtos.ResultResult;

import eu.dnetlib.data.proto.ResultResultProtos.ResultResult.Part;

import eu.dnetlib.data.proto.ResultResultProtos.ResultResult.PublicationDataset;

import eu.dnetlib.data.proto.ResultResultProtos.ResultResult.Similarity;

import eu.dnetlib.data.proto.ResultResultProtos.ResultResult.Supplement;

import eu.dnetlib.data.proto.TypeProtos.Type;

import eu.dnetlib.miscutils.collections.Pair;

import eu.dnetlib.miscutils.iterators.IterablePair;

public abstract class AbstractDNetXsltFunctions {

	public static final String URL_REGEX = "^(http|https|ftp)\\://.*";

	private static final int MAX_NSPREFIX_LEN = 12;

	public static Predicate<String> urlFilter = s -> s.trim().matches(URL_REGEX);

	public static Map<String, String> code2name = Maps.newHashMap();

	/*

	 * Obtained via COPY (select code, name from class) TO '/tmp/class_scheme.csv' (FORMAT csv, delimiter ',', FORCE_QUOTE *); on the

	 * relational db

	 */

	// code2name.put("openaire2.0_data","OpenAIRE Data (funded, referenced datasets)");

	static {

		code2name.put("MH", "Marshall Islands");

		code2name.put("CF", "Central African Republic");

		code2name.put("TD", "Chad");

		code2name.put("CN", "China (People's Republic of)");

		code2name.put("NG", "Nigeria");

		code2name.put("NF", "Norfolk Island");

		code2name.put("MP", "Northern Mariana Islands");

		code2name.put("PS", "Palestinian-administered areas");

		code2name.put("SZ", "Swaziland");

		code2name.put("max", "Manx");

		code2name.put("TW", "Taiwan");

		code2name.put("TJ", "Tajikistan");

		code2name.put("BSG", "Research for the benefit of specific groups");

		code2name.put("CP", "Collaborative project");

		code2name.put("12MONTHS", "12 Months Embargo");

		code2name.put("ace", "Achinese");

		code2name.put("egy", "Ancient Egyptian");

		code2name.put("ara", "Arabic");

		code2name.put("arc", "Aramaic");

		code2name.put("arp", "Arapaho");

		code2name.put("gon", "Gondi");

		code2name.put("ine", "Indo-European");

		code2name.put("ipk", "Inupiaq");

		code2name.put("ira", "Iranian");

		code2name.put("lim", "Limburgan; Limburger; Limburgish");

		code2name.put("mni", "Manipuri");

		code2name.put("mno", "Manobo");

		code2name.put("men", "Mende");

		code2name.put("CX", "Christmas Island");

		code2name.put("CC", "Cocos (Keeling) Islands");

		code2name.put("KM", "Comoros");

		code2name.put("CG", "Congo");

		code2name.put("CK", "Cook Islands");

		code2name.put("HR", "Croatia");

		code2name.put("arn", "Araucanian");

		code2name.put("art", "Artificial");

		code2name.put("nah", "Aztec");

		code2name.put("bug", "Buginese");

		code2name.put("chn", "Chinook jargon");

		code2name.put("chv", "Chuvash");

		code2name.put("mus", "Creek");

		code2name.put("mic", "Micmac");

		code2name.put("min", "Minangkabau");

		code2name.put("fro", "Old French");

		code2name.put("cpp", "Portuguese-based Creoles and Pidgins");

		code2name.put("som", "Somali");

		code2name.put("wen", "Sorbian");

		code2name.put("hrv", "Croatian");

		code2name.put("cus", "Cushitic");

		code2name.put("sot", "Sotho, Southern");

		code2name.put("sai", "South American Indian");

		code2name.put("esl/spa", "Spanish");

		code2name.put("CU", "Cuba");

		code2name.put("CW", "Curaçao");

		code2name.put("CZ", "Czech Republic");

		code2name.put("DK", "Denmark");

		code2name.put("ER", "Eritrea");

		code2name.put("TF", "French Southern Territories");

		code2name.put("GW", "Guinea-Bissau");

		code2name.put("VA", "Holy See (Vatican City State)");

		code2name.put("BO", "Bolivia");

		code2name.put("KY", "Cayman Islands");

		code2name.put("dra", "Dravidian");

		code2name.put("cpe", "English-based Creoles and Pidgins");

		code2name.put("oji", "Ojibwa");

		code2name.put("CIP-EIP-TN", "CIP-Eco-Innovation - CIP-Thematic Network");

		code2name.put("jav/jaw", "Javanese");

		code2name.put("ach", "Acoli");

		code2name.put("ada", "Adangme");

		code2name.put("afh", "Afrihili");

		code2name.put("afr", "Afrikaans");

		code2name.put("afa", "Afro-Asiatic");

		code2name.put("ale", "Aleut");

		code2name.put("alg", "Algonquian languages");

		code2name.put("arw", "Arawak");

		code2name.put("asm", "Assamese");

		code2name.put("ava", "Avaric");

		code2name.put("ave", "Avestan");

		code2name.put("bra", "Braj");

		code2name.put("bua", "Buriat");

		code2name.put("chr", "Cherokee");

		code2name.put("chy", "Cheyenne");

		code2name.put("jrb", "Judeo-Arabic");

		code2name.put("jpr", "Judeo-Persian");

		code2name.put("kab", "Kabyle");

		code2name.put("kac", "Kachin");

		code2name.put("kaa", "Kara-Kalpak");

		code2name.put("loz", "Lozi");

		code2name.put("mwr", "Marwari");

		code2name.put("DJ", "Djibouti");

		code2name.put("JM", "Jamaica");

		code2name.put("JP", "Japan");

		code2name.put("JE", "Jersey");

		code2name.put("JO", "Jordan");

		code2name.put("KZ", "Kazakhstan");

		code2name.put("KE", "Kenya");

		code2name.put("KI", "Kiribati");

		code2name.put("KR", "Korea (Republic of)");

		code2name.put("KP", "Korea, Democatric People's Republic of");

		code2name.put("XK", "Kosovo * UN resolution");

		code2name.put("KW", "Kuwait");

		code2name.put("NL", "Netherlands");

		code2name.put("PE", "Peru");

		code2name.put("PH", "Philippines");

		code2name.put("fre/fra", "French");

		code2name.put("PL", "Poland");

		code2name.put("PT", "Portugal");

		code2name.put("PR", "Puerto Rico");

		code2name.put("QA", "Qatar");

		code2name.put("RO", "Romania");

		code2name.put("RU", "Russian Federation");

		code2name.put("RW", "Rwanda");

		code2name.put("RE", "Réunion");

		code2name.put("sve/swe", "Swedish");

		code2name.put("myn", "Mayan");

		code2name.put("dum", "Middle Dutch");

		code2name.put("mun", "Munda");