D-Net

package eu.dnetlib.openaire.hadoop.utils;

import java.util.Arrays;

import java.util.Set;

import java.util.stream.Collectors;

import java.util.stream.Stream;

import com.google.common.collect.Sets;

import eu.dnetlib.data.proto.DatasourceOrganizationProtos.DatasourceOrganization.Provision;

import eu.dnetlib.data.proto.DedupProtos.Dedup;

import eu.dnetlib.data.proto.DedupSimilarityProtos.DedupSimilarity;

import eu.dnetlib.data.proto.PersonPersonProtos.PersonPerson.CoAuthorship;

import eu.dnetlib.data.proto.PersonResultProtos.PersonResult.Authorship;

import eu.dnetlib.data.proto.ProjectOrganizationProtos.ProjectOrganization.Participation;

import eu.dnetlib.data.proto.ProjectPersonProtos.ProjectPerson.ContactPerson;

import eu.dnetlib.data.proto.RelTypeProtos.RelType;

import eu.dnetlib.data.proto.RelTypeProtos.SubRelType;

import eu.dnetlib.data.proto.ResultOrganizationProtos.ResultOrganization.Affiliation;

import eu.dnetlib.data.proto.ResultProjectProtos.ResultProject.Outcome;

import eu.dnetlib.data.proto.ResultResultProtos.ResultResult.*;

import eu.dnetlib.data.proto.TypeProtos.Type;

/**

 * Common static utility methods to manage the hbase tables

 *

 * @author claudio

 */

public class HBaseTableUtils {

	private static final String SEPARATOR = "_";

	public static Set<String> listAllColumns() {

		final Set<String> union = Sets.union(listEntities(), listRelationships());

		return Sets.union(union, listDedupRelationships());

	}

	private static Set<String> listDedupRelationships() {

		final Set<String> cfs = Sets.newHashSet();

		cfs.add(RelType.organizationOrganization + SEPARATOR + SubRelType.dedup + SEPARATOR + Dedup.RelName.merges);

		cfs.add(RelType.organizationOrganization + SEPARATOR + SubRelType.dedup + SEPARATOR + Dedup.RelName.isMergedIn);

		cfs.add(RelType.organizationOrganization + SEPARATOR + SubRelType.dedupSimilarity + SEPARATOR + DedupSimilarity.RelName.isSimilarTo);

		cfs.add(RelType.personPerson + SEPARATOR + SubRelType.dedup + SEPARATOR + Dedup.RelName.merges);

		cfs.add(RelType.personPerson + SEPARATOR + SubRelType.dedup + SEPARATOR + Dedup.RelName.isMergedIn);

		cfs.add(RelType.personPerson + SEPARATOR + SubRelType.dedupSimilarity + SEPARATOR + DedupSimilarity.RelName.isSimilarTo);

		cfs.add(RelType.resultResult + SEPARATOR + SubRelType.dedup + SEPARATOR + Dedup.RelName.merges);

		cfs.add(RelType.resultResult + SEPARATOR + SubRelType.dedup + SEPARATOR + Dedup.RelName.isMergedIn);

		cfs.add(RelType.resultResult + SEPARATOR + SubRelType.dedupSimilarity + SEPARATOR + DedupSimilarity.RelName.isSimilarTo);

		return cfs;

	}

	private static Stream<String> types() {

		return Arrays.stream(Type.values())

				.map(Enum::toString);

	}

	public static Set<String> listEntities() {

		return types()

				.collect(Collectors.toSet());

	}

	public static Set<String> listRelationships() {

		final Set<String> cfs = Sets.newHashSet();

		cfs.add(RelType.datasourceOrganization + SEPARATOR + SubRelType.provision + SEPARATOR + Provision.RelName.isProvidedBy);

		cfs.add(RelType.datasourceOrganization + SEPARATOR + SubRelType.provision + SEPARATOR + Provision.RelName.provides);

		cfs.add(RelType.personPerson + SEPARATOR + SubRelType.coauthorship + SEPARATOR + CoAuthorship.RelName.isCoauthorOf);

		cfs.add(RelType.personResult + SEPARATOR + SubRelType.authorship + SEPARATOR + Authorship.RelName.isAuthorOf);

		cfs.add(RelType.personResult + SEPARATOR + SubRelType.authorship + SEPARATOR + Authorship.RelName.hasAuthor);

		cfs.add(RelType.projectOrganization + SEPARATOR + SubRelType.participation + SEPARATOR + Participation.RelName.hasParticipant);

		cfs.add(RelType.projectOrganization + SEPARATOR + SubRelType.participation + SEPARATOR + Participation.RelName.isParticipant);

		cfs.add(RelType.projectPerson + SEPARATOR + SubRelType.contactPerson + SEPARATOR + ContactPerson.RelName.isContact);

		cfs.add(RelType.projectPerson + SEPARATOR + SubRelType.contactPerson + SEPARATOR + ContactPerson.RelName.hasContact);

		cfs.add(RelType.resultProject + SEPARATOR + SubRelType.outcome + SEPARATOR + Outcome.RelName.isProducedBy);

		cfs.add(RelType.resultProject + SEPARATOR + SubRelType.outcome + SEPARATOR + Outcome.RelName.produces);

		cfs.add(RelType.resultResult + SEPARATOR + SubRelType.similarity + SEPARATOR + Similarity.RelName.hasAmongTopNSimilarDocuments);

		cfs.add(RelType.resultResult + SEPARATOR + SubRelType.similarity + SEPARATOR + Similarity.RelName.isAmongTopNSimilarDocuments);

		cfs.add(RelType.resultResult + SEPARATOR + SubRelType.supplement + SEPARATOR + Supplement.RelName.isSupplementedBy);

		cfs.add(RelType.resultResult + SEPARATOR + SubRelType.supplement + SEPARATOR + Supplement.RelName.isSupplementTo);

		cfs.add(RelType.resultResult + SEPARATOR + SubRelType.part + SEPARATOR + Part.RelName.isPartOf);

		cfs.add(RelType.resultResult + SEPARATOR + SubRelType.part + SEPARATOR + Part.RelName.hasPart);

		cfs.add(RelType.resultResult + SEPARATOR + SubRelType.publicationDataset + SEPARATOR + PublicationDataset.RelName.isRelatedTo);

		cfs.add(RelType.resultOrganization + SEPARATOR + SubRelType.affiliation + SEPARATOR + Affiliation.RelName.isAuthorInstitutionOf);

		cfs.add(RelType.resultOrganization + SEPARATOR + SubRelType.affiliation + SEPARATOR + Affiliation.RelName.hasAuthorInstitution);

		cfs.add(RelType.resultResult + SEPARATOR + SubRelType.version + SEPARATOR + SoftwareSoftware.RelName.isVersionOf);

		cfs.add(RelType.resultResult + SEPARATOR + SubRelType.relationship + SEPARATOR + Relationship.RelName.isRelatedTo);

		return cfs;

	}

}

package eu.dnetlib.openaire.exporter.model;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Splitter;

import com.google.common.collect.Lists;

import org.apache.commons.lang.StringUtils;

/**

 * Created by claudio on 20/09/16.

 */

public class Project {

	public static final String INFO_EU_REPO_GRANT_AGREEMENT = "info:eu-repo/grantAgreement/";

	private String code;

	private String acronym;

	private String title;

	private String call_identifier;

	private String startdate;

	private String enddate;

	private boolean ec_sc39;

	private boolean oa_mandate_for_publications;

	private boolean ec_article29_3;

	private String fundingpathid;

	private String description;

	private String jurisdiction;

	private String legalname;

	private String funder;

	private String countryclass;

	private String role;

	private String firstname;

	private String secondnames;

	private String email;

	public Project() {

	}

	public String getIdnamespace() {

		String res = INFO_EU_REPO_GRANT_AGREEMENT + getFunder()+"/";

		final String fundingProgram = asFundingProgram(getFundingpathid());

		if (StringUtils.isNotBlank(fundingProgram)) {

			res += fundingProgram;

		}

		res += "/" + escapeCode(getCode());

		if (StringUtils.isNotBlank(getJurisdiction())) {

			res += "/" + getJurisdiction();

		}

		return res;

	}

	public String getListLabel() {

		return String.format("for:value:component:_%s_project_id", asFunder(getFunder()));

	}

	private String asFunder(final String legalshortname) {

		switch (legalshortname.toLowerCase()) {

		case "ec":

			return asFundingProgram(getFundingpathid()).toLowerCase();

		default:

			return legalshortname.toLowerCase();

		}

	}

	public List<String> asList() {

		return Lists.newArrayList(

				clean(getCode()),

				clean(getAcronym()),

				clean(getTitle()),

				clean(getCall_identifier()),

				clean(getStartdate()),

				clean(getEnddate()),

				String.valueOf(isEc_article29_3()),

				String.valueOf(isOa_mandate_for_publications()),

				String.valueOf(isEc_article29_3()),

				clean(getDescription()),

				clean(getLegalname()),

				clean(getCountryclass()),

				clean(getRole()),

				clean(getFirstname()),

				clean(getSecondnames()),

				clean(getEmail()));

	}

	private String clean(final String s) {

		return StringUtils.isNotBlank(s) ? "\"" + s.replaceAll("\\n|\\t|\\s+", " ").replace("\"","\"\"").trim() + "\"" : "";

	}

	private String escapeCode(final String code) {

		return replaceSlash(code);

	}

	private String asFundingProgram(final String fundingpathid) {

		final ArrayList<String> strings = Lists.newArrayList(Splitter.on("::").split(fundingpathid));

		if(strings.size() <= 1) throw new IllegalStateException("Unexpected funding id: "+fundingpathid);

		if(strings.size() == 2) return "";

		else return replaceSlash(strings.get(2));

	}

	private String replaceSlash(final String s) {

		return s.replaceAll("/", "%2F");

	}

	public String getCode() {

		return code;

	}

	public Project setCode(final String code) {

		this.code = code;

		return this;

	}

	public String getAcronym() {

		return acronym;

	}

	public Project setAcronym(final String acronym) {

		this.acronym = acronym;

		return this;

	}

	public String getTitle() {

		return title;

	}

	public Project setTitle(final String title) {

		this.title = title;

		return this;

	}

	public String getCall_identifier() {

		return call_identifier;

	}

	public Project setCall_identifier(final String call_identifier) {

		this.call_identifier = call_identifier;

		return this;

	}

	public String getStartdate() {

		return startdate;

	}

	public Project setStartdate(final String startdate) {

		this.startdate = startdate;

		return this;

	}

	public String getEnddate() {

		return enddate;

	}

	public Project setEnddate(final String enddate) {

		this.enddate = enddate;

		return this;

	}

	public boolean isEc_sc39() {

		return ec_sc39;

	}

	public Project setEc_sc39(final boolean ec_sc39) {

		this.ec_sc39 = ec_sc39;

		return this;

	}

	public boolean isOa_mandate_for_publications() {

		return oa_mandate_for_publications;

	}

	public Project setOa_mandate_for_publications(final boolean oa_mandate_for_publications) {

		this.oa_mandate_for_publications = oa_mandate_for_publications;

		return this;

	}

	public boolean isEc_article29_3() {

		return ec_article29_3;

	}

	public Project setEc_article29_3(final boolean ec_article29_3) {

		this.ec_article29_3 = ec_article29_3;

		return this;

	}

	public String getFundingpathid() {

		return fundingpathid;

	}

	public Project setFundingpathid(final String fundingpathid) {

		this.fundingpathid = fundingpathid;

		return this;

	}

	public String getDescription() {

		return description;

	}

	public Project setDescription(final String description) {

		this.description = description;

		return this;

	}

	public String getJurisdiction() {

		return jurisdiction;

	}

	public Project setJurisdiction(final String jurisdiction) {

		this.jurisdiction = jurisdiction;

		return this;

	}

	public String getLegalname() {

		return legalname;

	}

	public Project setLegalname(final String legalname) {

		this.legalname = legalname;

		return this;

	}

	public String getCountryclass() {

		return countryclass;

	}

	public Project setCountryclass(final String countryclass) {

		this.countryclass = countryclass;

		return this;

	}

	public String getRole() {

		return role;

	}

	public Project setRole(final String role) {

		this.role = role;

		return this;

	}

	public String getFirstname() {

		return firstname;

	}

	public Project setFirstname(final String firstname) {

		this.firstname = firstname;

		return this;

	}

	public String getSecondnames() {

		return secondnames;

	}

	public Project setSecondnames(final String secondnames) {

		this.secondnames = secondnames;

		return this;

	}

	public String getEmail() {

		return email;

	}

	public Project setEmail(final String email) {

		this.email = email;

		return this;

	}

	public String getFunder() {

		return funder;

	}

	public Project setFunder(final String funder) {

		this.funder = funder;

		return this;

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-openaireplus-mapping-utils</artifactId>

	<packaging>jar</packaging>

	<version>6.0.4</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.0.4</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>com.google.guava</groupId>

			<artifactId>guava</artifactId>

			<version>${google.guava.version}</version>

		</dependency>		

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>commons-codec</groupId>

			<artifactId>commons-codec</artifactId>

			<version>${commons.codec.version}</version>

		</dependency>

		<dependency>

			<groupId>dom4j</groupId>

			<artifactId>dom4j</artifactId>

			<version>${dom4j.version}</version>

			<exclusions>

				<exclusion>

					<artifactId>xml-apis</artifactId>

					<groupId>xml-apis</groupId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>net.sf.supercsv</groupId>

			<artifactId>super-csv</artifactId>

			<version>2.4.0</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-openaire-data-protos</artifactId>

			<version>[3.7.8]</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-pace-core</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-misc-utils</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-hadoop-commons</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-index-solr-common</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>	

		<dependency>

			<groupId>com.googlecode.protobuf-java-format</groupId>

			<artifactId>protobuf-java-format</artifactId>

			<version>1.2</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-openaireplus-profiles</artifactId>

			<version>[1.0.0,2.0.0)</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.apache.commons</groupId>

			<artifactId>commons-lang3</artifactId>

			<version>3.5</version>

		</dependency>

	</dependencies>

</project>

package eu.dnetlib.data.transform.xml;

import java.util.List;

import eu.dnetlib.data.transform.xml.OpenTrialsXsltFunctions.JsonProv;

import org.junit.After;

import org.junit.Before;

import org.junit.Test;

import static org.junit.Assert.assertEquals;

/**

 * OpenTrialsXsltFunctions Tester.

 *

 */

public class OpenTrialsXsltFunctionsTest {

	private String jsonProv = "[{\"url\" : \"http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT00378508\", \"sourceID\" : \"77eb42c5-0ec7-4e31-963a-5736b66f2d26\", \"sourceName\" : \"ictrp\"},{\"url\" : \"https://www.clinicaltrials.gov/ct2/show/NCT00378508?lup_e=02%2F04%2F2016&lup_s=01%2F01%2F2001&rank=175075&resultsxml=true\", \"sourceID\" : \"b389497c-0833-432b-a09b-930526b7b4d4\", \"sourceName\" : \"nct\"}]";

	private String jsonProvWithNull = "[{\"url\" : \"\", \"sourceID\" : \"77eb42c5-0ec7-4e31-963a-5736b66f2d26\", \"sourceName\" : \"ictrp\"},{\"url\" : \"https://www.clinicaltrials.gov/ct2/show/NCT00378508?lup_e=02%2F04%2F2016&lup_s=01%2F01%2F2001&rank=175075&resultsxml=true\", \"sourceID\" : \"b389497c-0833-432b-a09b-930526b7b4d4\", \"sourceName\" : \"nct\"}]";

	private String jidentifiers = "{112683,NCT00920439}";

	private String jsonRecord = "[{\"source_id\" : \"nct\", \"source_url\" : \"https://clinicaltrials.gov/ct2/show/NCT02321059\", \"is_primary\" : true},{\"source_id\" : \"ictrp\", \"source_url\" : \"http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT02321059\", \"is_primary\" : false}]";

	private String jsonRecordNull = "[{\"source_id\" : \"nct\", \"source_url\" : \"https://clinicaltrials.gov/ct2/show/NCT02321059\"},{\"source_id\" : \"ictrp\", \"source_url\" : \"http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT02321059\", \"is_primary\" : false}]";

	private String jsonRecordVoid = "[{\"source_id\" : \"\", \"source_url\" : \"\", \"is_primary\" : \"\"}]";

	private String jsonRecondPrimary = "[{\"source_id\" : \"nct\", \"source_url\" : \"https://clinicaltrials.gov/ct2/show/NCT02321059\", \"is_primary\" : false},{\"source_id\" : \"ictrp\", \"source_url\" : \"http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT02321059\", \"is_primary\" : false}]";

	private String jsonPeopleVoid ="[{\"person_name\" : null, \"person_id\" : null, \"person_role\" : null}]";

	private String jsonPeopleOne = "[{\"person_name\" : \"Henk Verheul, M.D., PhD\", \"person_id\" : \"116438e9-f8b1-46e5-a1f8-20f851cab73c\", \"person_role\" : \"principal_investigator\"}]";

	private String jsonPeopleMore = "[{\"person_name\" : \"Henk Verheul, M.D., PhD\", \"person_id\" : \"116438e9-f8b1-46e5-a1f8-20f851cab73c\", \"person_role\" : \"principal_investigator\"},{\"person_name\" : \"Miriam Pippolippo Baglioni, PhD\", \"person_id\" : \"fake\", \"person_role\" : \"principal_investigator\"}]";

	private String jsonOrganizationVoid = "[{\"organization_name\" : null, \"organization_id\" : null, \"organization_role\" : null}]";

	private String jsonOrganizationOne = "[{\"organization_name\" : \"Södertälje sjukhus AB\", \"organization_id\" : \"15f0d004-b82b-408c-8605-38a57352468d\", \"organization_role\" : \"sponsor\"}]";

	private String jsonOrganizationMore = "[{\"organization_name\" : \"Södertälje sjukhus AB\", \"organization_id\" : \"15f0d004-b82b-408c-8605-38a57352468d\", \"organization_role\" : \"sponsor\"},{\"organization_name\" : \"Miriam Baglioni AB\", \"organization_id\" : \"fake\", \"organization_role\" : \"primary_sponsor\"}]";

	private String jsonLocationVoid = "[{\"location_name\" : null}]";

	private String jsonLocationOne = "[{\"location_name\" : \"China\"}]";

	private String jsonLocationMore = "[{\"location_name\" : \"China\"},{\"location_name\" : \"North Korea\"}]";

	@Before

	public void before() throws Exception {

	}

	@After

	public void after() throws Exception {

	}

	/**

	 * Method: getProvs(String jsonProvList)

	 */

	@Test

	public void testGetProvs() throws Exception {

		List<JsonProv> list = OpenTrialsXsltFunctions.getProvs(jsonProv);

		assertEquals(2, list.size());

	}

	/**

	 * Method: getMainIdentifierURL(String jsonProvList)

	 */

	@Test

	public void testGetMainIdentifierURL() throws Exception {

		String url = OpenTrialsXsltFunctions.getMainIdentifierURL(jsonProv);

		assertEquals( "http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT00378508", url );

		url = OpenTrialsXsltFunctions.getMainIdentifierURL(jsonProvWithNull);

		assertEquals("https://www.clinicaltrials.gov/ct2/show/NCT00378508?lup_e=02%2F04%2F2016&lup_s=01%2F01%2F2001&rank=175075&resultsxml=true", url);

	}

	@Test

	public void testGetPrimaryRecordUrl(){

		String url = OpenTrialsXsltFunctions.getPrimaryRecordUrl(jsonRecord);

		assertEquals("https://clinicaltrials.gov/ct2/show/NCT02321059", url);

	}

	@Test

	public void testGetPrimaryRecordID(){

		String id = OpenTrialsXsltFunctions.getPrimaryRecordIdentifier(jsonRecord);

		assertEquals("nct", id);

	}

	@Test

	public void testGetPrimaryRecordUrlNull(){

		String url = OpenTrialsXsltFunctions.getPrimaryRecordUrl(jsonRecordNull);

		assertEquals("https://clinicaltrials.gov/ct2/show/NCT02321059", url);

	}

	@Test

	public void testGetPrimaryRecordUrlVoid(){

		String url = OpenTrialsXsltFunctions.getPrimaryRecordUrl(jsonRecordVoid);

		assertEquals("", url);

	}

	@Test

	public void testGetPrimaryRecordUrlNoPrimary(){

		String url = OpenTrialsXsltFunctions.getPrimaryRecordUrl(jsonRecondPrimary);

		assertEquals("https://clinicaltrials.gov/ct2/show/NCT02321059", url);

	}

	@Test

	public void testGetPrimaryRecordIDNoPrimary(){

		String id = OpenTrialsXsltFunctions.getPrimaryRecordIdentifier(jsonRecondPrimary);

		assertEquals("nct", id);

	}

	@Test

	public void testGetPrincipalInvestigatorsVoid(){

		String url = OpenTrialsXsltFunctions.getPrincipalInvestigators(jsonPeopleVoid);

		assertEquals("",url);

	}

	@Test

	public void testGetPrincipalInvestigatorsOne(){

		String url = OpenTrialsXsltFunctions.getPrincipalInvestigators(jsonPeopleOne);

		assertEquals("Verheul, Henk", url);

	}

	@Test

	public void testGetPrincipalInvestigatorsMore(){

		String url = OpenTrialsXsltFunctions.getPrincipalInvestigators(jsonPeopleMore);

		assertEquals("Verheul, Henk@@Baglioni, Miriam Pippolippo", url);

	}

	@Test

	public void testgGetTrialOrganizationsVoid(){

		String url = OpenTrialsXsltFunctions.getTrialOrganizations(jsonOrganizationVoid);

		assertEquals("",url);

	}

	@Test

	public void testgGetTrialOrganizationsOne(){

		String url = OpenTrialsXsltFunctions.getTrialOrganizations(jsonOrganizationOne);

		assertEquals("Södertälje sjukhus AB@sponsor", url);

	}

	@Test

	public void testgGetTrialOrganizationsMore(){

		String url = OpenTrialsXsltFunctions.getTrialOrganizations(jsonOrganizationMore);

		assertEquals("Södertälje sjukhus AB@sponsor@@Miriam Baglioni AB@sponsor", url);

	}

	@Test

	public void testgGetTrialLocationsVoid(){

		String url = OpenTrialsXsltFunctions.getTrialLocations(jsonLocationVoid);

		assertEquals("",url);

	}

	@Test

	public void testgGetTrialLocationsOne(){

		String url = OpenTrialsXsltFunctions.getTrialLocations(jsonLocationOne);

		assertEquals("China", url);

	}

	@Test

	public void testgGetTrialLocationsMore(){

		String url = OpenTrialsXsltFunctions.getTrialLocations(jsonLocationMore);

		assertEquals("China@@North Korea", url);

	}

	@Test

	public void testGetNotPrimaryRecordUrlPrimary(){

		String url = OpenTrialsXsltFunctions.getNotPrimaryRecordUrl(jsonRecondPrimary);

		assertEquals("http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT02321059", url);

	}

	@Test

	public void testGetNotPrimaryRecordUrlVoid(){

		String url = OpenTrialsXsltFunctions.getNotPrimaryRecordUrl(jsonRecordVoid);

		assertEquals("", url);

	}

	@Test

	public void testGetNotPrimaryRecordUrl(){

		String url = OpenTrialsXsltFunctions.getNotPrimaryRecordUrl(jsonRecord);

		assertEquals("http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT02321059", url);

	}

}

package eu.dnetlib.pace.model.gt;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import eu.dnetlib.data.proto.FieldTypeProtos.DataInfo;

import eu.dnetlib.data.proto.FieldTypeProtos.Qualifier;

import eu.dnetlib.data.proto.FieldTypeProtos.StringField;

import eu.dnetlib.data.proto.KindProtos.Kind;

import eu.dnetlib.data.proto.OafProtos.Oaf;

import eu.dnetlib.data.proto.OafProtos.OafEntity;

import eu.dnetlib.data.proto.PersonProtos.Person;

import eu.dnetlib.data.proto.PersonProtos.Person.CoAuthor;

import eu.dnetlib.data.proto.PersonProtos.Person.MergedPerson;

import eu.dnetlib.data.proto.PersonProtos.Person.Metadata;

import eu.dnetlib.data.proto.TypeProtos.Type;

import org.apache.commons.lang.StringUtils;

public class GTAuthorMapper {

	public Oaf map(final GTAuthor gta) {

		final Oaf.Builder oaf = Oaf.newBuilder();

		oaf.setDataInfo(getDataInfo());

		oaf.setLastupdatetimestamp(System.currentTimeMillis());

		oaf.setKind(Kind.entity);

		oaf.setEntity(getOafEntity(gta));

		return oaf.build();

	}

	private OafEntity getOafEntity(final GTAuthor gta) {

		final OafEntity.Builder entity = OafEntity.newBuilder();

		entity.setType(Type.person);

		entity.setId(gta.getId());

		entity.setPerson(getPerson(gta));

		return entity.build();

	}

	private Person getPerson(final GTAuthor gta) {

		final Person.Builder person = Person.newBuilder();

		if (gta.getAuthor() != null) {

			final Person.Metadata.Builder m = Person.Metadata.newBuilder();

			if (StringUtils.isNotBlank(gta.getAuthor().getFullname())) {

				m.setFullname(sf(gta.getAuthor().getFullname()));

			}

			if (StringUtils.isNotBlank(gta.getAuthor().getFirstname())) {

				m.setFirstname(sf(gta.getAuthor().getFirstname()));

			}

			if (StringUtils.isNotBlank(gta.getAuthor().getSecondnames())) {

				m.addSecondnames(sf(gta.getAuthor().getSecondnames()));

			}

			person.setMetadata(m.build());

		}

		person.setAnchor(true);

		person.addAllMergedperson(Iterables.transform(gta.getMerged(), new Function<Author, MergedPerson>() {

			@Override

			public MergedPerson apply(final Author a) {

				final MergedPerson.Builder mp = MergedPerson.newBuilder();

				mp.setId(a.getId());

				mp.setMetadata(getMetadata(a));

				return mp.build();

			}

		}));

		person.addAllCoauthor(Iterables.transform(gta.getCoAuthors(), new Function<eu.dnetlib.pace.model.gt.CoAuthor, CoAuthor>() {

			@Override

			public CoAuthor apply(final eu.dnetlib.pace.model.gt.CoAuthor ca) {

				final CoAuthor.Builder coAuthor = CoAuthor.newBuilder();

				coAuthor.setId(ca.getId());

				if (StringUtils.isNotBlank(ca.getAnchorId())) {

					coAuthor.setAnchorId(ca.getAnchorId());

				}

				coAuthor.setMetadata(getMetadata(ca));

				return coAuthor.build();

			}

		}));

		return person.build();

	}

	private Metadata getMetadata(final Author a) {

		final Metadata.Builder m = Metadata.newBuilder();

		m.setFullname(sf(a.getFullname()));

		if (a.isWellFormed()) {

			m.setFirstname(sf(a.getFirstname()));

			m.addSecondnames(sf(a.getSecondnames()));

		}

		return m.build();

	}

	private DataInfo getDataInfo() {

		final DataInfo.Builder d = DataInfo.newBuilder();

		d.setDeletedbyinference(false);

		d.setInferred(true);

		d.setTrust("0.5");

		d.setInferenceprovenance("dedup-person-groundtruth");

		d.setProvenanceaction(Qualifier.newBuilder().setClassid("").setClassname("").setSchemeid("").setSchemename(""));

		return d.build();

	}

	private StringField sf(final String s) {

		final StringField.Builder sf = StringField.newBuilder();

		sf.setValue(s);

		return sf.build();

	}

}

package eu.dnetlib.data.mapreduce.util;

import static org.junit.Assert.assertFalse;

import static org.junit.Assert.assertNotNull;

import java.util.List;

import org.junit.Test;

import eu.dnetlib.data.proto.KindProtos.Kind;

import eu.dnetlib.miscutils.functional.xml.IndentXmlString;

public class OafDecoderTest {

	@Test

	public void testAsXml() {

		final OafDecoder decoder = OafTest.embed(OafTest.getResult("50|id_1"), Kind.entity);

		assertNotNull(decoder);

		assertNotNull(decoder.asXml());

		System.out.println(IndentXmlString.apply(decoder.asXml()));

	}

	@Test

	public void testGetFieldValues() {

		final OafDecoder decoder = OafTest.embed(OafTest.getResult("50|id_1"), Kind.entity);

		final String path = "result/metadata/title/value";

		final List<String> titles = decoder.decodeEntity().getFieldValues(path);

		assertNotNull(titles);

		assertFalse(titles.isEmpty());

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"

	xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:dr="http://www.driver-repository.eu/namespace/dr"

	xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:oaa="http://namespace.openaire.eu/oaa"

	xmlns:oaf="http://namespace.openaire.eu/oaf" xmlns:dnet="eu.dnetlib.data.transform.xml.OafToHbaseXsltFunctions"

	xmlns:exslt="http://exslt.org/common"

	xmlns:action="http://namespace.openaire.eu/action" version="1.0"

	extension-element-prefixes="exslt"

	exclude-result-prefixes="xsl dc dr dri oaa oaf dnet exslt">

	<xsl:output omit-xml-declaration="yes" indent="yes" />

	<xsl:param name="trust" select="string('0.9')" />

	<xsl:param name="provenance" select="string('UNKNOWN')" />

	<xsl:template match="/*">

		<xsl:choose>

			<xsl:when test="count(//RELATION) = 0">

				<ACTIONS />

			</xsl:when>

			<xsl:otherwise>

				<ACTIONS>

					<xsl:for-each select="//RELATION">

						<xsl:choose>

							<xsl:when test="./@type = 'resultProject'">

								<xsl:variable name="resultId" select="./@source" />

								<xsl:variable name="projectId">

									<xsl:choose>

										<xsl:when test="starts-with(@target, '40|')">

											<xsl:value-of select="./@target" />

										</xsl:when>

										<xsl:otherwise>

											<xsl:value-of select="dnet:oafSplitId('project', normalize-space(@target))"/>

										</xsl:otherwise>

									</xsl:choose>

								</xsl:variable>

								<ACTION targetKey="{$resultId}" targetColumnFamily="resultProject_outcome_isProducedBy" targetColumn="{$projectId}">

									<xsl:value-of select="dnet:rel($resultId, $projectId, 'resultProject', 'outcome', 'isProducedBy', $provenance, $trust)" />

								</ACTION>

								<ACTION targetKey="{$projectId}" targetColumnFamily="resultProject_outcome_produces" targetColumn="{$resultId}">

									<xsl:value-of select="dnet:rel($projectId, $resultId, 'resultProject', 'outcome', 'produces', $provenance, $trust)" />

								</ACTION>

							</xsl:when>

							<!-- TODO: check this block, it caused problems:

									Cannot convert argument/return type in call to method 'eu.dnetlib.actionmanager.actions.infopackage.DMFInfoPackageToHbaseXsltFunctions.oafPersonResultFromInfoPackage(node-set, node-set, result-tree, reference, reference)'

									FATAL ERROR:  'Could not compile stylesheet'

							<xsl:when test="./@type = 'personResult'">

								<xsl:variable name="personId" select="./@source" />

								<xsl:variable name="resultId" select="./@target" />

								<xsl:variable name="pos">

									<xsl:choose>

										<xsl:when test="@position"><xsl:value-of select="@position"/></xsl:when>

										<xsl:otherwise>1000</xsl:otherwise>

									</xsl:choose>

								</xsl:variable>

								<ACTION targetKey="{$personId}" targetColumnFamily="personResult" targetColumn="{$resultId}">

									<xsl:value-of select="dnet:oafPersonResultFromInfoPackage($personId, $resultId, $pos, $provenance, $trust)" />

								</ACTION>

								<ACTION targetKey="{$resultId}" targetColumnFamily="personResult" targetColumn="{$personId}">

									<xsl:value-of select="dnet:oafPersonResultFromInfoPackage($resultId, $personId, $pos, $provenance, $trust)" />

								</ACTION>

							</xsl:when>

							-->

						</xsl:choose>

					</xsl:for-each>

				</ACTIONS>

			</xsl:otherwise>

		</xsl:choose>

	</xsl:template>

</xsl:stylesheet>

package eu.dnetlib.openaire.exporter.model;

import java.io.IOException;

import java.io.StringReader;

import com.google.common.collect.Lists;

import org.apache.commons.io.IOUtils;

import org.springframework.core.io.ClassPathResource;

import org.springframework.core.io.Resource;

import static org.junit.Assert.assertEquals;

/**

 * Created by claudio on 22/09/16.

 */

public class ProjectDetailTest {

	private Resource projectsCsv = new ClassPathResource("/eu/dnetlib/openaire/exporter/model/projectDetails.csv");

	private Resource projectsJson = new ClassPathResource("/eu/dnetlib/openaire/exporter/model/projectDetails.json");

	//@Test

	public void testSerialisationCSV() throws IOException {

		final String csv = IOUtils.toString(projectsCsv.getInputStream());

		doTest(csv, "csv");

	}

	//@Test

	public void testSerialisationCSV2() throws IOException {

		final ProjectDetail p = new ProjectDetail()

				.setAcronym("acro")

				.setCode("01")

				.setOptional1("op1")

				.setOptional2("op2")

				.setProjectId("project_01")

				.setJsonextrainfo("extraInfo")

				.setFundingPath(Lists.newArrayList("fundingpath1", "fundingpath2"));

		doTest(p.asCSV(), "csv");

	}

	//@Test

	public void testSerialisationJSON() throws IOException {

		final String json = IOUtils.toString(projectsJson.getInputStream());

		doTest(json, "json");

	}

	private void doTest(final String data, final String format) throws IOException {

		final StringReader reader = new StringReader(data);

		for(String line : IOUtils.readLines(reader)) {

			System.out.println("line:          " + line);

			ProjectDetail p;

			String s = "";

			switch (format) {

			case "csv":

				p = ProjectDetail.fromCSV(line);

				s = p.asCSV();

				break;

			case "json":

				p = ProjectDetail.fromJson(line);

				s = p.asJson();

				break;

			default: throw new IllegalArgumentException("invalid format: " + format);

			}

			System.out.println("serialisation: " + s);

			assertEquals(s, line);

		}

	}

}

package eu.dnetlib.pace.model;

import static org.junit.Assert.assertFalse;

import static org.junit.Assert.assertTrue;

import org.junit.Test;

import com.google.common.collect.Iterables;

import com.google.common.collect.Sets;

import com.google.common.collect.Sets.SetView;

import eu.dnetlib.pace.AbstractProtoPaceTest;

import eu.dnetlib.pace.config.Config;

public class ProtoDocumentBuilderTest extends AbstractProtoPaceTest {

	@Test

	public void test_serialise1() {

		final String id = "12345";

		final Config config = getResultFullConf();

		final MapDocument document = ProtoDocumentBuilder.newInstance(id, getResult(id), config.model());

		assertFalse(document.fieldNames().isEmpty());

		assertFalse(Iterables.isEmpty(document.fields()));

		System.out.println("original:\n" + document);

		final String stringDoc = MapDocumentSerializer.toString(document);

		System.out.println("srialization:\n" + stringDoc);

		final MapDocument decoded = MapDocumentSerializer.decode(stringDoc.getBytes());

		final SetView<String> diff = Sets.difference(document.fieldNames(), decoded.fieldNames());

		assertTrue(diff.isEmpty());

		System.out.println("decoded:\n" + decoded);

	}

	@Test

	public void test_serialise2() {

		final String id = "12345";

		final String path = "/eu/dnetlib/pace/model/gt.author.manghi1.json";

		final Config config = getPersonConf();

		final MapDocument document = ProtoDocumentBuilder.newInstance(id, getPersonGT(path).getEntity(), config.model());

		assertFalse(document.fieldNames().isEmpty());

		assertFalse(Iterables.isEmpty(document.fields()));

		System.out.println("original:\n" + document);

		final String stringDoc = MapDocumentSerializer.toString(document);

		System.out.println("srialization:\n" + stringDoc);

		final MapDocument decoded = MapDocumentSerializer.decode(stringDoc.getBytes());

		final SetView<String> diff = Sets.difference(document.fieldNames(), decoded.fieldNames());

		assertTrue(diff.isEmpty());

		System.out.println("decoded:\n" + decoded);

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"

                xmlns:oaf="http://namespace.openaire.eu/oaf" xmlns:dc="http://purl.org/dc/elements/1.1/"

                xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:dr="http://www.driver-repository.eu/namespace/dr"

                xmlns:dnet="eu.dnetlib.data.transform.xml.OafToHbaseXsltFunctions"

                xmlns:exslt="http://exslt.org/common"

                version="1.0" extension-element-prefixes="exslt"

                exclude-result-prefixes="xsl oaf dr dri dnet exslt">

	<xsl:output omit-xml-declaration="yes" indent="yes"/>

	<xsl:param name="writeCoAuthors" select="false()"/>

	<xsl:param name="trust" select="string('0.9')"/>

	<xsl:param name="provenance" select="string('sysimport:crosswalk:repository')"/>

	<xsl:template match="/*">

		<xsl:variable name="about" select="/*[local-name() = 'record']/*[local-name() = 'about']"/>

		<xsl:variable name="dateofcollection" select="/*[local-name() = 'record']/*[local-name() = 'header']/*[local-name()='dateOfCollection' and text()][1]"/>

		<xsl:variable name="dateoftransformation" select="/*[local-name() = 'record']/*[local-name() = 'header']/*[local-name()='dateOfTransformation' and text()][1]"/>

		<xsl:variable name="metadata" select="exslt:node-set(/*[local-name() = 'record']/*[local-name()='metadata']/*)"/>

		<xsl:variable name="namespaceprefix">

			<xsl:choose>

				<!-- TODO check namespaceprefix length is 12 -->

				<xsl:when test="string-length(//oaf:datasourceprefix) &gt; 0">

					<xsl:value-of select="//oaf:datasourceprefix"/>

				</xsl:when>

				<xsl:otherwise>

					<xsl:value-of select="unknown_"/>

				</xsl:otherwise>

			</xsl:choose>

		</xsl:variable>

		<xsl:choose>

			<!-- 			<xsl:when test="count($metadata) = 0 or string-length($namespaceprefix) = 0"> -->

			<xsl:when test="count($metadata) = 0">

				<ROWS/>

			</xsl:when>

			<xsl:otherwise>

				<xsl:variable name="objIdentifier" select="//dri:objIdentifier"/>

				<xsl:variable name="resultId" select="dnet:oafSimpleId('result', //dri:objIdentifier)"/>

				<xsl:if test="string-length($resultId) &gt; 0">

					<xsl:variable name="collectedfromid" select="dnet:oafSplitId('datasource', //oaf:collectedFrom/@id)"/>

					<xsl:variable name="collectedfromname" select="//oaf:collectedFrom/@name"/>

					<xsl:variable name="hostedbyid" select="dnet:oafSplitId('datasource', //oaf:hostedBy/@id)"/>

					<xsl:variable name="hostedbyname" select="//oaf:hostedBy/@name"/>

					<xsl:variable name="originalidTest" select="/*[local-name()='record']/*[local-name() = 'header']/*[local-name() = 'recordIdentifier']"/>

					<xsl:variable name="originalid">

						<xsl:choose>

							<xsl:when test="contains($originalidTest, '::')">

								<xsl:value-of select="substring-after($originalidTest, '::')"/>

							</xsl:when>

							<xsl:otherwise>

								<xsl:value-of select="$originalidTest"/>

							</xsl:otherwise>

						</xsl:choose>

					</xsl:variable>

					<xsl:variable name="result"

					              select="dnet:oafResult($resultId, $provenance, $trust, $about, $hostedbyid, $hostedbyname, $collectedfromid,

					              $collectedfromname, $originalid, $dateofcollection, $dateoftransformation, $metadata)"/>

					<ROWS>

						<ROW key="{$resultId}" columnFamily="result">

							<QUALIFIER name="body" type="base64">

								<xsl:value-of select="$result"/>

							</QUALIFIER>

						</ROW>

						<xsl:for-each select="//*[local-name()='projectid']">

							<xsl:variable name="projectId" select="dnet:oafSplitId('project', normalize-space(.))"/>

							<xsl:if test="string-length($projectId) &gt; 0">

								<xsl:variable name="resultproject"

								              select="dnet:rel($resultId, $projectId, 'resultProject', 'outcome', 'isProducedBy',

								              $collectedfromid, $collectedfromname, $provenance, $trust, $about)"/>

								<xsl:variable name="projectresult"

								              select="dnet:rel($projectId, $resultId, 'resultProject', 'outcome', 'produces',

								              $collectedfromid, $collectedfromname, $provenance, $trust, $about)"/>

								<ROW key="{$resultId}" columnFamily="resultProject_outcome_isProducedBy">

									<QUALIFIER name="{$projectId}" type="base64">

										<xsl:value-of select="$resultproject"/>

									</QUALIFIER>

								</ROW>

								<ROW key="{$projectId}" columnFamily="resultProject_outcome_produces">

									<QUALIFIER name="{$resultId}" type="base64">

										<xsl:value-of select="$projectresult"/>

									</QUALIFIER>

								</ROW>

							</xsl:if>

						</xsl:for-each>

						<xsl:for-each select="//*[local-name()='relatedDataset']">

							<!-- relatedDataset ids must be in the openaire format  -->

							<xsl:variable name="datasetId" select="dnet:oafSimpleId('result', normalize-space(./@id))"/>

							<xsl:if test="string-length($datasetId) &gt; 0">

								<xsl:variable name="resultDataset"

								              select="dnet:rel($resultId, $datasetId, 'resultResult', 'publicationDataset', 'isRelatedTo',

								              $collectedfromid, $collectedfromname, $provenance, $trust, $about)"/>

								<xsl:variable name="datasetResult"

								              select="dnet:rel($datasetId, $resultId, 'resultResult', 'publicationDataset', 'isRelatedTo',

								              $collectedfromid, $collectedfromname, $provenance, $trust, $about)"/>

								<ROW key="{$resultId}" columnFamily="resultResult_publicationDataset_isRelatedTo">

									<QUALIFIER name="{$datasetId}" type="base64">

										<xsl:value-of select="$resultDataset"/>

									</QUALIFIER>

								</ROW>

								<ROW key="{$datasetId}" columnFamily="resultResult_publicationDataset_isRelatedTo">

									<QUALIFIER name="{$resultId}" type="base64">

										<xsl:value-of select="$datasetResult"/>

									</QUALIFIER>

								</ROW>

							</xsl:if>

						</xsl:for-each>

						<xsl:for-each select="//*[local-name()='creator']">

							<xsl:if test="string-length(normalize-space(.)) &gt; 0">

								<xsl:variable name="originalPersonId">

									<xsl:choose>

										<xsl:when test="string-length(@nameIdentifier) &gt; 0">

											<xsl:value-of select="normalize-space(@nameIdentifier)"/>

										</xsl:when>

										<xsl:otherwise>

											<xsl:value-of select="concat($originalid, '::', normalize-space(.))"/>

										</xsl:otherwise>

									</xsl:choose>

								</xsl:variable>

								<xsl:variable name="personId" select="dnet:oafId('person', $namespaceprefix, $originalPersonId)"/>

								<xsl:variable name="position" select="position()"/>

								<xsl:variable name="person"

								              select="dnet:person($personId, $about, $provenance, $trust, $collectedfromid, $collectedfromname,

			                                  $originalPersonId, $dateofcollection, $dateoftransformation, normalize-space(.), @nameIdentifier, @nameIdentifierScheme)"/>

								<xsl:variable name="personresult"

								              select="dnet:personResult_Authorship($personId, $resultId, $position, 'isAuthorOf',

								              $collectedfromid, $collectedfromname, $provenance, $trust, $about)"/>

								<xsl:variable name="resultperson"

								              select="dnet:personResult_Authorship($resultId, $personId, $position, 'hasAuthor',

								              $collectedfromid, $collectedfromname, $provenance, $trust, $about)"/>

								<xsl:if test="string-length($personId) &gt; 0">

									<ROW key="{$personId}" columnFamily="person">

										<QUALIFIER name="body" type="base64">

											<xsl:value-of select="$person"/>

										</QUALIFIER>

									</ROW>

									<ROW key="{$personId}" columnFamily="personResult_authorship_isAuthorOf">

										<QUALIFIER name="{$resultId}" type="base64">

											<xsl:value-of select="$personresult"/>

										</QUALIFIER>

									</ROW>

									<ROW key="{$resultId}" columnFamily="personResult_authorship_hasAuthor">

										<QUALIFIER name="{$personId}" type="base64">

											<xsl:value-of select="$resultperson"/>

										</QUALIFIER>

									</ROW>

								</xsl:if>

								<!-- COAUTHORS -->

								<xsl:if test="$writeCoAuthors = true()">

									<xsl:for-each select="../dc:creator">

										<xsl:if test="$position != position()">

											<xsl:variable name="originalCoauthorId">

												<xsl:choose>

													<xsl:when test="string-length(@nameIdentifier) &gt; 0">

														<xsl:value-of select="normalize-space(@nameIdentifier)"/>

													</xsl:when>

													<xsl:otherwise>

														<xsl:value-of select="concat($originalid, '::', normalize-space(.))"/>

													</xsl:otherwise>

												</xsl:choose>

											</xsl:variable>

											<xsl:variable name="coauthorId"

											              select="dnet:oafId('person', $namespaceprefix, $originalCoauthorId)"/>

											<xsl:variable name="personperson"

											              select="dnet:rel($personId, $coauthorId, 'personPerson', 'coauthorship', 'isCoAuthorOf',

											              $collectedfromid, $collectedfromname, $provenance, $trust, $about)"/>

											<ROW key="{$personId}" columnFamily="personPerson_coAuthorship_isCoAuthorOf">

												<QUALIFIER name="{$coauthorId}" type="base64">

													<xsl:value-of select="$personperson"/>

												</QUALIFIER>

											</ROW>

										</xsl:if>

									</xsl:for-each>

								</xsl:if>

								<!-- / COAUTHORS -->

							</xsl:if>

						</xsl:for-each>

					</ROWS>

				</xsl:if>

			</xsl:otherwise>

		</xsl:choose>

	</xsl:template>

</xsl:stylesheet>

package eu.dnetlib.openaire.exporter.model;

import org.junit.Before;

import org.junit.Test;

import static org.junit.Assert.assertEquals;

public class ProjectTest {

	Project pMZOS;

	Project pFP7;

	@Before

	public void setUp() throws Exception {

		pMZOS = new Project()

				.setFunder("MZOS")

				.setJurisdiction("HR")

				.setFundingpathid("irb_hr______::MZOS")

				.setAcronym("")

				.setTitle("Project Title")

				.setCode("115-1152437-2500")

				.setStartdate("2007-01-01")

				.setEnddate("2009-01-01");

		pFP7 = new Project()

				.setFunder("EC")

				.setJurisdiction("EU")

				.setFundingpathid("ec__________::EC::FP7::SP1::NMP")

				.setAcronym("REFFIBRE")

				.setTitle("Project Title")

				.setCode("604187")

				.setStartdate("2013-11-01")

				.setEnddate("20015-01-01");

	}

	@Test

	public void testIdNamespaceMZOS(){

		String ns = pMZOS.getIdnamespace();

		assertEquals("info:eu-repo/grantAgreement/MZOS//115-1152437-2500/HR", ns);

	}

	@Test

	public void testIdNamespaceFP7(){

		String ns = pFP7.getIdnamespace();

		assertEquals("info:eu-repo/grantAgreement/EC/FP7/604187/EU", ns);

	}

}

package eu.dnetlib.pace.model.gt;

import static org.junit.Assert.assertTrue;

import java.util.Set;

import org.junit.Test;

import com.google.common.collect.Sets;

import eu.dnetlib.pace.model.gt.Author;

import eu.dnetlib.pace.model.gt.Authors;

public class AuthorTest {

	@Test

	public void test() {

		final Set<Author> s1 = getAuthors(3);

		final Set<Author> s2 = getAuthors(3);

		final Set<Author> i = Sets.intersection(s1, s2);

		System.out.println(i);

		assertTrue(i.size() == 3);

	}

	@Test

	public void test1() {

		final Authors a1 = new Authors(a("1", "Wang, M."));

		final Authors a2 = new Authors(a("1", "Wang, M."));

		final Set<Author> i = Sets.intersection(a1, a2);

		assertTrue(i.size() == 1);

	}

	private Set<Author> getAuthors(final int n) {

		final Set<Author> s = Sets.newHashSet();

		for (int i = 0; i < n; i++) {

			s.add(a(i + "", "name" + i));

		}

		return s;

	}

	private Author a(final String id, final String fullname) {

		final Author a = new Author();

		a.setId(id);

		a.setFullname(fullname);

		return a;

	}

}

nih_________::3R01GM073898-02S1,,3R01GM073898-02S1,23188 $,,"{""orgname"":""UNIVERSITY OF CALIFORNIA SAN DIEGO"", ""activity"":""R01"", ""administeringic"":""GM"", ""serialnumber"":""73898"", ""coreprojectnum"":""R01GM073898""}","[""\u003cfundingtree\u003e\n      \u003cfunder\u003e\n         \u003cid\u003enih_________::NIH\u003c/id\u003e\n         \u003cshortname\u003eNIH\u003c/shortname\u003e\n         \u003cname\u003eNational Institutes of Health\u003c/name\u003e\n         \u003cjurisdiction\u003eUS\u003c/jurisdiction\u003e\n      \u003c/funder\u003e\n      \u003cfunding_level_0\u003e\n         \u003cid\u003enih_________::NIH::NATIONAL_INSTITUTE_OF_GENERAL_MEDICAL_SCIENCES\u003c/id\u003e\n         \u003cname\u003eNATIONAL INSTITUTE OF GENERAL MEDICAL SCIENCES\u003c/name\u003e\n         \u003cdescription\u003eNATIONAL INSTITUTE OF GENERAL MEDICAL SCIENCES\u003c/description\u003e\n         \u003cparent/\u003e\n         \u003cclass\u003enih:fundingStream\u003c/class\u003e\n      \u003c/funding_level_0\u003e\n   \u003c/fundingtree\u003e""]"

corda_______::100202,RECOMP,100202,JTI-CP-ARTEMIS,http://cordis.europa.eu/fp7/home_en.html,{},"[""\u003cfundingtree\u003e\u003cfunder\u003e\u003cid\u003eec__________::EC\u003c/id\u003e\u003cshortname\u003eEC\u003c/shortname\u003e\u003cname\u003eEuropean Commission\u003c/name\u003e\u003cjurisdiction\u003eEU\u003c/jurisdiction\u003e\u003c/funder\u003e\u003cfunding_level_2\u003e\u003cid\u003eec__________::EC::FP7::SP1::SP1-JTI\u003c/id\u003e\u003cdescription\u003eJoint Technology Initiatives (Annex IV-SP1)\u003c/description\u003e\u003cname\u003eSP1-JTI\u003c/name\u003e\u003cclass\u003eec:program\u003c/class\u003e\u003cparent\u003e\u003cfunding_level_1\u003e\u003cid\u003eec__________::EC::FP7::SP1\u003c/id\u003e\u003cdescription\u003eSP1-Cooperation\u003c/description\u003e\u003cname\u003eSP1\u003c/name\u003e\u003cclass\u003eec:specificprogram\u003c/class\u003e\u003cparent\u003e\u003cfunding_level_0\u003e\u003cid\u003eec__________::EC::FP7\u003c/id\u003e\u003cdescription\u003eSEVENTH FRAMEWORK PROGRAMME\u003c/description\u003e\u003cname\u003eFP7\u003c/name\u003e\u003cparent/\u003e\u003cclass\u003eec:frameworkprogram\u003c/class\u003e\u003c/funding_level_0\u003e\u003c/parent\u003e\u003c/funding_level_1\u003e\u003c/parent\u003e\u003c/funding_level_2\u003e\u003c/fundingtree\u003e""]"

corda__h2020::633080,MACC-III,633080,SPACE,SPACE,{},"[""\u003cfundingtree\u003e\u003cfunder\u003e\u003cid\u003eec__________::EC\u003c/id\u003e\u003cshortname\u003eEC\u003c/shortname\u003e\u003cname\u003eEuropean Commission\u003c/name\u003e\u003cjurisdiction\u003eEU\u003c/jurisdiction\u003e\u003c/funder\u003e\u003cfunding_level_1\u003e\u003cid\u003eec__________::EC::H2020::CSA\u003c/id\u003e\u003cdescription\u003eCoordination and support action\u003c/description\u003e\u003cname\u003eCSA\u003c/name\u003e\u003cclass\u003eec:h2020toas\u003c/class\u003e\u003cparent\u003e\u003cfunding_level_0\u003e\u003cid\u003eec__________::EC::H2020\u003c/id\u003e\u003cname\u003eH2020\u003c/name\u003e\u003cdescription\u003eHorizon 2020 Framework Programme\u003c/description\u003e\u003cparent/\u003e\u003cclass\u003eec:h2020fundings\u003c/class\u003e\u003c/funding_level_0\u003e\u003c/parent\u003e\u003c/funding_level_1\u003e\u003c/fundingtree\u003e""]"

nsf_________::0000096,,0000096,,,{},"[""\u003cfundingtree\u003e\u003cfunder\u003e\u003cid\u003ensf_________::NSF\u003c/id\u003e\u003cshortname\u003eNSF\u003c/shortname\u003e\u003cname\u003eNational Science Foundation\u003c/name\u003e\u003cjurisdiction\u003eUS\u003c/jurisdiction\u003e\u003c/funder\u003e\u003cfunding_level_1\u003e\u003cid\u003ensf_________::NSF::OD::OD/OIA\u003c/id\u003e\u003cdescription\u003eOffice of Integrative Activities\u003c/description\u003e\u003cname\u003eOffice of Integrative Activities\u003c/name\u003e\u003cparent\u003e\u003cfunding_level_0\u003e\u003cid\u003ensf_________::NSF::OD\u003c/id\u003e\u003cdescription\u003eOffice of the Director\u003c/description\u003e\u003cname\u003eOffice of the Director\u003c/name\u003e\u003cparent/\u003e\u003cclass\u003ensf:fundingStream\u003c/class\u003e\u003c/funding_level_0\u003e\u003c/parent\u003e\u003c/funding_level_1\u003e\u003c/fundingtree\u003e""]"

fct_________::100107,PTDC/SAU-ESA/100107/2008,100107,,,{},"[""\u003cfundingtree\u003e\u003cfunder\u003e\u003cid\u003efct_________::FCT\u003c/id\u003e\u003cshortname\u003eFCT\u003c/shortname\u003e\u003cname\u003eFundação para a Ciência e a Tecnologia, I.P.\u003c/name\u003e\u003cjurisdiction\u003ePT\u003c/jurisdiction\u003e\u003c/funder\u003e\u003cfunding_level_0\u003e\u003cid\u003efct_________::FCT::5876-PPCDTI\u003c/id\u003e\u003cdescription\u003e5876-PPCDTI\u003c/description\u003e\u003cname\u003e5876-PPCDTI\u003c/name\u003e\u003cparent/\u003e\u003cclass\u003efct:program\u003c/class\u003e\u003c/funding_level_0\u003e\u003c/fundingtree\u003e""]"

{ 

	"wf" : { 

        "threshold" : "0.99", 

        "dedupRun" : "001", 

        "entityType" : "result", 

        "orderField" : "title", 

        "queueMaxSize" : "2000",

        "groupMaxSize" : "10",

        "slidingWindowSize" : "200",

        "rootBuilder" : [ "result", "personResult_authorship_hasAuthor", "resultProject_outcome_isProducedBy", "resultResult_publicationDataset_isRelatedTo", "resultResult_similarity_isAmongTopNSimilarDocuments", "resultResult_similarity_hasAmongTopNSimilarDocuments" ],

        "includeChildren" : "true" 

    },

	"pace" : {		

		"clustering" : [

			{ "name" : "ngrampairs", "fields" : [ "title" ], "params" : { "max" : "1", "ngramLen" : "3"} },

			{ "name" : "suffixprefix", "fields" : [ "title" ], "params" : { "max" : "1", "len" : "3" } } 

		],

		"strictConditions" : [

			{ "name" : "exactMatch", "fields" : [ "pid" ] }

		], 

  		"conditions" : [ 

  			{ "name" : "titleVersionMatch", "fields" : [ "title" ] },

  			{ "name" : "sizeMatch", "fields" : [ "authors" ] }

  		],		

		"model" : [

			{ "name" : "pid", "algo" : "Null", "type" : "String", "weight" : "0.0", "ignoreMissing" : "true", "path" : "pid[qualifier#classid = {doi}]/value", "overrideMatch" : "true" }, 	

			{ "name" : "title", "algo" : "LevensteinTitle", "type" : "String", "weight" : "1.0", "ignoreMissing" : "false", "path" : "result/metadata/title[qualifier#classid = {main title}]/value" },

			{ "name" : "authors", "algo" : "Null", "type" : "String", "weight" : "0.0", "ignoreMissing" : "true", "path" : "result/author/metadata/fullname/value" }

		],

		"blacklists" : { 

			"title" : [

				"^THE ASSOCIATION AND THE GENERAL MEDICAL COUNCIL$",

				"^Problems with perinatal pathology\.?$",

				"(?i)^Cases? of Puerperal Convulsions$",

				"(?i)^Operative Gyna?ecology$",

				"(?i)^Mind the gap\!?\:?$",

				"^Chronic fatigue syndrome\.?$",

				"^Cartas? ao editor Letters? to the Editor$",

				"^Note from the Editor$",

				"^Anesthesia Abstract$",

				"^Annual report$",

				"(?i)^“?THE RADICAL PREVENTION OF VENEREAL DISEASE\.?”?$",

				"(?i)^Graph and Table of Infectious Diseases?$",

				"^Presentation$",

				"(?i)^Reviews and Information on Publications$",

				"(?i)^PUBLIC HEALTH SERVICES?$",

				"(?i)^COMBINED TEXT-?BOOK OF OBSTETRICS AND GYN(Æ|ae)COLOGY$",

				"(?i)^Adrese autora$",

				"(?i)^Systematic Part .*\. Catalogus Fossilium Austriae, Band 2: Echinoidea neogenica$",

				"(?i)^Acknowledgement to Referees$",

				"(?i)^Behçet's disease\.?$",

				"(?i)^Isolation and identification of restriction endonuclease.*$",

				"(?i)^CEREBROVASCULAR DISEASES?.?$",

				"(?i)^Screening for abdominal aortic aneurysms?\.?$",

				"^Event management$",

				"(?i)^Breakfast and Crohn's disease.*\.?$",

				"^Cálculo de concentraciones en disoluciones acuosas. Ejercicio interactivo\..*\.$",

				"(?i)^Genetic and functional analyses of SHANK2 mutations suggest a multiple hit model of Autism spectrum disorders?\.?$",

				"^Gushi hakubutsugaku$",

				"^Starobosanski nadpisi u Bosni i Hercegovini \(.*\)$",							

				"^Intestinal spirocha?etosis$",

				"^Treatment of Rodent Ulcer$",

				"(?i)^\W*Cloud Computing\W*$",

				"^Compendio mathematico : en que se contienen todas las materias mas principales de las Ciencias que tratan de la cantidad$",				

				"^Free Communications, Poster Presentations: Session [A-F]$",

				"^“The Historical Aspects? of Quackery\.?”$",

				"^A designated centre for people with disabilities operated by St John of God Community Services (Limited|Ltd), Louth$",

				"^P(er|re)-Mile Premiums for Auto Insurance\\.?$",

				"(?i)^Case Report$",							

				"^Boletín Informativo$",

				"(?i)^Glioblastoma Multiforme$",

				"(?i)^Nuevos táxones animales descritos en la península Ibérica y Macaronesia desde 1994 \\(.*\\)$",

				"^Zaměstnanecké výhody$",

				"(?i)^The Economics of Terrorism and Counter-Terrorism: A Survey \\(Part .*\\)$",

				"(?i)^Carotid body tumours?\\.?$", 

				"(?i)^\\[Españoles en Francia : La condición Emigrante.*\\]$",

				"^Avant-propos$",

				"(?i)^St\. Patrick's Cathedral, Dublin, County Dublin - Head(s)? and Capital(s)?$",

				"(?i)^St\. Patrick's Cathedral, Dublin, County Dublin - Bases?$",

				"(?i)^PUBLIC HEALTH VERSUS THE STATE$",							

				"^Viñetas de Cortázar$",

				"(?i)^Search for heavy neutrinos and W(\\[|_|\\(|_\\{|-)?R(\\]|\\)|\\})? bosons with right-handed couplings in a left-right symmetric model in pp collisions at.*TeV(\.)?$",