D-Net

{

		"type_source": "SVN",

		"goal": "package -U source:jar",

		"url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet50/modules/dnet-graph-domain/trunk/",

		"deploy_repository": "dnet5-snapshots",

		"version": "5",

		"mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it",

		"deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet5-snapshots",

		"name": "dnet-graph-domain"

}

package eu.dnetlib.pace.distance;

import static org.junit.Assert.assertTrue;

import java.util.ArrayList;

import java.util.List;

import eu.dnetlib.pace.distance.eval.ScoreResult;

import org.junit.Test;

import com.google.common.collect.Lists;

import eu.dnetlib.pace.AbstractProtoPaceTest;

import eu.dnetlib.pace.config.Config;

import eu.dnetlib.pace.model.MapDocument;

public class DetectorTest extends AbstractProtoPaceTest {

	@Test

	public void testScoreResult() {

		final Config config = getResultProdConf();

		final MapDocument resA = result(config, "A", "Recent results from CDFsd");

		final MapDocument resB = result(config, "B", "Recent results from CDF");

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		System.out.println(sr.toString());

	}

	@Test

	public void testDistanceResultSimple() {

		final Config config = getResultSimpleConf();

		final MapDocument resA = result(config, "A", "Recent results from CDF");

		final MapDocument resB = result(config, "B", "Recent results from CDF");

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		final double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		assertTrue(d == 1.0);

	}

	@Test

	public void testDistanceResultSimpleMissingDates() {

		final Config config = getResultSimpleConf();

		final MapDocument resA = result(config, "A", "Recent results from BES");

		final MapDocument resB = result(config, "A", "Recent results from CES");

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		final double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		assertTrue(d > 0.97);

	}

	@Test

	public void testDistanceResultInvalidDate() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "title title title 6BESR", "2013-01-05");

		final MapDocument resB = result(config, "B", "title title title 6BESR", "qwerty");

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		final double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		assertTrue(d == 1.0);

	}

	@Test

	public void testDistanceResultMissingOneDate() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "title title title 6BESR", null);

		final MapDocument resB = result(config, "B", "title title title 6CLER", "2012-02");

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		assertTrue((d > 0.9) && (d < 1.0));

	}

	@Test

	public void testDistanceResult() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "title title title BES", "");

		final MapDocument resB = result(config, "B", "title title title CLEO");

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		assertTrue((d > 0.9) && (d < 1.0));

	}

	@Test

	public void testDistanceResultMissingTwoDate() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "title title title 6BESR");

		final MapDocument resB = result(config, "B", "title title title 6CLER");

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		assertTrue((d > 0.9) && (d < 1.0));

	}

	@Test

	public void testDistanceOrganizationIgnoreMissing() {

		final Config config = getOrganizationSimpleConf();

		final MapDocument orgA = organization(config, "A", "CONSIGLIO NAZIONALE DELLE RICERCHE");

		final MapDocument orgB = organization(config, "B", "CONSIGLIO NAZIONALE DELLE RICERCHE", "CNR");

		final ScoreResult sr = new PaceDocumentDistance().between(orgA, orgB, config);

		final double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		assertTrue(d == 1.0);

	}

	@Test

	public void testDistanceResultCase1() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "Search the Standard Model Higgs boson", "2003");

		final MapDocument resB = result(config, "B", "Search for the Standard Model Higgs Boson", "2003");

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		assertTrue((d > 0.9) && (d < 1.0));

	}

	@Test

	public void testDistanceResultCaseDoiMatch1() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "Search the Standard Model Higgs boson", "2003", "10.1594/PANGAEA.726855");

		final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", "10.1594/PANGAEA.726855");

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		assertTrue("exact DOIs will produce an exact match", d == 1.0);

	}

	@Test

	public void testDistanceResultCaseDoiMatch2() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1594/PANGAEA.726855");

		final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2005", "10.1594/PANGAEA.726855");

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		assertTrue("exact DOIs will produce an exact match, regardless of different titles or publication years", d == 1.0);

	}

	@Test

	public void testDistanceResultCaseDoiMatch3() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1016/j.jmb.2010.12.024");

		final MapDocument resB = result(config, "B", "Conference proceedings on X. Appendix", "2003");

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		assertTrue("a missing DOI will casue the comparsion to continue with the following conditions", d == 1.0);

	}

	@Test

	public void testDistanceResultCaseDoiMatch4() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1016/j.jmb.2010.12.024");

		final MapDocument resB = result(config, "B", "Conference proceedings on X. Appendix", "2005");

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		assertTrue("a missing DOI, comparsion continues with the following conditions, different publication years will drop the score to 0", d == 0.0);

	}

	@Test

	public void testDistanceResultCaseDoiMatch5() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "Search for the Standard Model Higgs Boson", "2003", "10.1016/j.jmb.2010.12.020");

		final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003");

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		assertTrue("a missing DOI, comparsion continues with the following conditions", (d > 0.9) && (d < 1.0));

	}

	@Test

	public void testDistanceResultCaseDoiMatch6() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "Conference proceedings on X. Appendix", "2003", "10.1016/j.jmb.2010.12.024");

		final MapDocument resB = result(config, "B", "Conference proceedings on X. Appendix", "2003", "anotherDifferentDOI");

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		assertTrue("different DOIs will drop the score to 0, regardless of the other fields", d == 0.0);

	}

	@Test

	public void testDistanceResultCaseDoiMatch7() {

		final Config config = getResultConf();

		final MapDocument resA = result(config, "A", "Adrenal Insufficiency asd asd", "1951", Lists.newArrayList("PMC2037944", "axdsds"));

		final MapDocument resB = result(config, "B", "Adrenal Insufficiency", "1951", "PMC2037944");

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		assertTrue("different DOIs will drop the score to 0, regardless of the other fields", d > 0.9 & d < 1);

	}

	// http://dx.doi.org/10.1594/PANGAEA.726855 doi:10.1594/PANGAEA.726855

	@Test

	public void testDistanceResultCaseAuthor1() {

		final Config config = getResultAuthorsConf();

		final List<String> authorsA = Lists.newArrayList("a", "b", "c", "d");

		final List<String> authorsB = Lists.newArrayList("a", "b", "c");

		final List<String> pid = Lists.newArrayList();

		final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA);

		final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB);

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		final double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		assertTrue(d == 0.0);

	}

	@Test

	public void testDistanceResultCaseAuthor2() {

		final Config config = getResultAuthorsConf();

		final List<String> authorsA = Lists.newArrayList("a", "b", "c");

		final List<String> authorsB = Lists.newArrayList("a", "b", "c");

		final List<String> pid = Lists.newArrayList();

		final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA);

		final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB);

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		final double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		assertTrue(d == 1.0);

	}

	@Test

	public void testDistanceResultCaseAuthor3() {

		final Config config = getResultAuthorsConf();

		final List<String> authorsA = Lists.newArrayList("Bardi, A.", "Manghi, P.", "Artini, M.");

		final List<String> authorsB = Lists.newArrayList("Bardi Alessia", "Manghi Paolo", "Artini Michele");

		final List<String> pid = Lists.newArrayList();

		final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA);

		final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB);

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		assertTrue((d > 0.9) && (d < 1.0));

	}

	@Test

	public void testDistanceResultCaseAuthor4() {

		final Config config = getResultAuthorsConf();

		final List<String> authorsA = Lists.newArrayList("Bardi, Alessia", "Manghi, Paolo", "Artini, Michele", "a");

		final List<String> authorsB = Lists.newArrayList("Bardi Alessia", "Manghi Paolo", "Artini Michele");

		final List<String> pid = Lists.newArrayList();

		final MapDocument resA = result(config, "A", "Search the Standard Model Higgs Boson", "2003", pid, authorsA);

		final MapDocument resB = result(config, "B", "Search the Standard Model Higgs Boson", "2003", pid, authorsB);

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		final double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		// assertTrue(d.getScore() == 0.0);

	}

	@Test

	public void testDistanceResultFullConf() {

		final Config config = getResultFullConf();

		final List<String> authorsA = Lists.newArrayList("Nagarajan Pranesh", "Guy Vautier", "Punyanganie de Silva");

		final List<String> authorsB = Lists.newArrayList("Pranesh Nagarajan", "Vautier Guy", "de Silva Punyanganie");

		final MapDocument resA =

				result(config, "A", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010",

						"10.1186/1752-1947-4-299", authorsA);

		final MapDocument resB =

				result(config, "B", "Presentations of perforated colonic pathology in patients with polymyalgia rheumatica: two case reports", "2010",

						"10.1186/1752-1947-4-299", authorsB);

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		final double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		// assertTrue(d.getScore() == 0.0);

	}

	@Test

	public void testDistanceProdConf1() {

		final Config config = getResultProdConf();

		final MapDocument resA =

				result(config,

						"A",

						" Analysis of Transfer Embryo-Derived de-duplication");

		final MapDocument resB =

				result(config,

						"B",

						" Analysis of Transfer Embryo Derived deduplication");

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		final double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		// assertTrue(d.getScore() == 0.0);

	}

	@Test

	public void testDistanceProdConf2() {

		final Config config = getResultProdConf();

		final MapDocument resA =

				result(config,

						"A",

						"qwerty aaabbbbbbbb bbb ccc ddddd");

		final MapDocument resB =

				result(config,

						"B",

						"qwert aaabbbbbbbb bbb ccc ddddd");

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		final double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		// assertTrue(d.getScore() == 0.0);

	}

	@Test

	public void testDistanceProdConf3() {

		final Config config = getResultProdConf();

		final List<String> authorsA = Lists.newArrayList("Bardi, Alessia", "Manghi, Paolo", "Artini, Michele", "a");

		final List<String> authorsB = Lists.newArrayList("Bardi Alessia", "Manghi Paolo", "Artini Michele");

		final List<String> pid = Lists.newArrayList();

		final MapDocument resA =

				result(config,

						"A",

						"qwerty aaabbbbbbbb bbb ccc ddddd", "2003", pid, authorsA);

		final MapDocument resB =

				result(config,

						"B",

						"qwert aaabbbbbbbb bbb ccc ddddd", "2003", pid, authorsB);

		final ScoreResult sr = new PaceDocumentDistance().between(resA, resB, config);

		final double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		assertTrue(d == 0.0);

	}

	@Test

	public void testDistancePersonConf1() {

		final Config config = getPersonConf();

		final MapDocument p1 = person(config, "p1_id", getPersonGT("/eu/dnetlib/pace/model/gt.author.manghi1.fo.json"));

		final MapDocument p2 = person(config, "p2_id", getPersonGT("/eu/dnetlib/pace/model/gt.author.manghi2.fo.json"));

		final ScoreResult sr = new PaceDocumentDistance().between(p1, p2, config);

		final double d = sr.getScore();

		System.out.println(String.format(" d ---> %s", d));

		// assertTrue(d.getScore() == 0.0);

	}

}

package eu.dnetlib.pace.clustering;

import org.junit.Before;

import org.junit.Test;

import eu.dnetlib.pace.AbstractProtoPaceTest;

import eu.dnetlib.pace.config.Config;

import eu.dnetlib.pace.config.Type;

import eu.dnetlib.pace.model.FieldListImpl;

import eu.dnetlib.pace.model.FieldValueImpl;

import eu.dnetlib.pace.model.MapDocument;

public class ClusteringCombinerTest extends AbstractProtoPaceTest {

	private Config config;

	@Before

	public void setUp() {

		config = getResultFullConf();

	}

	@Test

	public void testCombine() {

		String title = "Dipping in Cygnus X-2 in a multi-wavelength campaign due to absorption of extended ADC emission";

		MapDocument result = result(config, "A", title, "2013");

		FieldListImpl fl = new FieldListImpl();

		fl.add(new FieldValueImpl(Type.String, "desc", "lorem ipsum cabalie qwerty"));

		result.getFieldMap().put("desc", fl);

		System.out.println(title);

		System.out.println(ClusteringCombiner.combine(result, config));

	}

}

package eu.dnetlib.pace.clustering;

import org.junit.Before;

import org.junit.Test;

import eu.dnetlib.pace.AbstractProtoPaceTest;

import eu.dnetlib.pace.config.Config;

import eu.dnetlib.pace.config.Type;

import eu.dnetlib.pace.model.FieldListImpl;

import eu.dnetlib.pace.model.FieldValueImpl;

import eu.dnetlib.pace.model.MapDocument;

public class BlacklistAwareClusteringCombinerTest extends AbstractProtoPaceTest {

	private Config config;

	@Before

	public void setUp() {

		config = getResultFullConf();

	}

	@Test

	public void testCombine() {

		final MapDocument result =

				result(config, "A", "Dipping in Cygnus X-2 in a multi-wavelength campaign due to absorption of extended ADC emission", "2013");

		final FieldListImpl fl = new FieldListImpl();

		fl.add(new FieldValueImpl(Type.String, "desc", "hello world description pipeline"));

		result.getFieldMap().put("desc", fl);

		fl.clear();

		fl.add(new FieldValueImpl(Type.String, "title", "lorem ipsum cabalie qwerty"));

		final FieldListImpl field = (FieldListImpl) result.getFieldMap().get("title");

		field.add(fl);

		System.out.println(BlacklistAwareClusteringCombiner.filterAndCombine(result, config, config.blacklists()));

	}

}

package eu.dnetlib.pace;

import java.io.IOException;

import java.io.StringWriter;

import java.util.ArrayList;

import java.util.List;

import com.google.common.collect.Lists;

import com.google.gson.Gson;

import eu.dnetlib.data.mapreduce.util.DNGFTest;

import eu.dnetlib.data.proto.DNGFProtos.DNGF;

import eu.dnetlib.data.proto.DNGFProtos.DNGFEntity;

import eu.dnetlib.data.proto.FieldTypeProtos.Qualifier;

import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty;

import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty.Builder;

import eu.dnetlib.data.proto.OrganizationProtos.Organization;

import eu.dnetlib.data.proto.PersonProtos.Person;

import eu.dnetlib.data.proto.PublicationProtos.Publication;

import eu.dnetlib.data.proto.TypeProtos;

import eu.dnetlib.pace.config.Config;

import eu.dnetlib.pace.config.DedupConfig;

import eu.dnetlib.pace.config.Type;

import eu.dnetlib.pace.model.Field;

import eu.dnetlib.pace.model.FieldValueImpl;

import eu.dnetlib.pace.model.MapDocument;

import eu.dnetlib.pace.model.ProtoDocumentBuilder;

import eu.dnetlib.pace.model.gt.GTAuthor;

import eu.dnetlib.pace.model.gt.GTAuthorMapper;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.RandomStringUtils;

import org.apache.commons.lang3.StringUtils;

public abstract class AbstractProtoPaceTest extends DNGFTest {

	protected DedupConfig getResultFullConf() {

		return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.full.pace.conf"));

	}

	protected DedupConfig getResultSimpleConf() {

		return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.simple.pace.conf"));

	}

	protected DedupConfig getResultConf() {

		return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.pace.conf"));

	}

	protected DedupConfig getOrganizationSimpleConf() {

		return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/organization.pace.conf"));

	}

	protected DedupConfig getResultAuthorsConf() {

		return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.authors.pace.conf"));

	}

	protected DedupConfig getPersonConf() {

		return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/person.pace.conf"));

	}

	protected DedupConfig getResultProdConf() {

		return DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/result.prod.pace.conf"));

	}

	protected MapDocument person(final Config conf, final String id, final DNGF oaf) {

		return ProtoDocumentBuilder.newInstance(id, oaf.getEntity(), conf.model());

	}

	protected DNGF getPersonGT(final String path) {

		return new GTAuthorMapper().map(getGTAuthor(path));

	}

	protected GTAuthor getGTAuthor(final String path) {

		final Gson gson = new Gson();

		final String json = readFromClasspath(path);

		final GTAuthor gta = gson.fromJson(json, GTAuthor.class);

		return gta;

	}

	private String readFromClasspath(final String filename) {

		final StringWriter sw = new StringWriter();

		try {

			IOUtils.copy(getClass().getResourceAsStream(filename), sw);

			return sw.toString();

		} catch (final IOException e) {

			throw new RuntimeException("cannot load resource from classpath: " + filename);

		}

	}

	protected MapDocument result(final Config config, final String id, final String title) {

		return result(config, id, title, null, new ArrayList<String>(), null);

	}

	protected MapDocument result(final Config config, final String id, final String title, final String date) {

		return result(config, id, title, date, new ArrayList<String>(), null);

	}

	protected MapDocument result(final Config config, final String id, final String title, final String date, final List<String> pid) {

		return result(config, id, title, date, pid, null);

	}

	protected MapDocument result(final Config config, final String id, final String title, final String date, final String pid) {

		return result(config, id, title, date, pid, null);

	}

	protected MapDocument result(final Config config, final String id, final String title, final String date, final String pid, final List<String> authors) {

		return result(config, id, title, date, Lists.newArrayList(pid), authors);

	}

	protected MapDocument result(final Config config, final String id, final String title, final String date, final List<String> pid, final List<String> authors) {

		final Publication.Metadata.Builder metadata = Publication.Metadata.newBuilder();

		if (!StringUtils.isBlank(title)) {

			metadata.addTitle(getStruct(title, getQualifier("main title", "dnet:titles")));

			metadata.addTitle(getStruct(RandomStringUtils.randomAlphabetic(10), getQualifier("alternative title", "dnet:titles")));

		}

		if (!StringUtils.isBlank(date)) {

			metadata.setDateofacceptance(sf(date));

		}

		final DNGFEntity.Builder entity = dngfEntity(id, TypeProtos.Type.publication);

		final Publication.Builder result = Publication.newBuilder().setMetadata(metadata);

		if (authors != null) {

			for (final String author : authors) {

				result.addAuthor(person(author));

			}

		}

		entity.setPublication(result);

		if (pid != null) {

			for(String p : pid) {

				if (!StringUtils.isBlank(p)) {

					entity.addPid(sp(p, "doi"));

					//entity.addPid(sp(RandomStringUtils.randomAlphabetic(10), "oai"));

				}

			}

		}

		final DNGFEntity build = entity.build();

		return ProtoDocumentBuilder.newInstance(id, build, config.model());

	}

	private Person.Builder person(final String author) {

		final Person.Builder person = Person.newBuilder();

		final eu.dnetlib.pace.model.Person p = new eu.dnetlib.pace.model.Person(author, false);

		final Person.Metadata.Builder metadata = Person.Metadata.newBuilder();

		if (p.isAccurate()) {

			metadata.setFirstname(sf(p.getNormalisedFirstName()));

			metadata.addSecondnames(sf(p.getNormalisedSurname()));

			metadata.setFullname(sf(p.getNormalisedFullname()));

		} else {

			metadata.setFullname(sf(p.getOriginal()));

		}

		return person.setMetadata(metadata);

	}

	private DNGFEntity.Builder dngfEntity(final String id, final eu.dnetlib.data.proto.TypeProtos.Type type) {

		final DNGFEntity.Builder entity = DNGFEntity.newBuilder().setId(id).setType(type);

		return entity;

	}

	protected MapDocument organization(final Config config, final String id, final String legalName) {

		return organization(config, id, legalName, null);

	}

	protected MapDocument organization(final Config config, final String id, final String legalName, final String legalShortName) {

		final Organization.Metadata.Builder metadata = Organization.Metadata.newBuilder();

		if (legalName != null) {

			metadata.setLegalname(sf(legalName));

		}

		if (legalShortName != null) {

			metadata.setLegalshortname(sf(legalShortName));

		}

		final DNGFEntity.Builder entity = dngfEntity(id, TypeProtos.Type.publication);

		entity.setOrganization(Organization.newBuilder().setMetadata(metadata));

		return ProtoDocumentBuilder.newInstance(id, entity.build(), config.model());

	}

	private StructuredProperty sp(final String pid, final String type) {

		final Builder pidSp =

				StructuredProperty.newBuilder().setValue(pid)

						.setQualifier(Qualifier.newBuilder().setClassid(type).setClassname(type).setSchemeid("dnet:pid_types").setSchemename("dnet:pid_types"));

		return pidSp.build();

	}

	protected Field title(final String s) {

		return new FieldValueImpl(Type.String, "title", s);

	}

	protected static StructuredProperty.Builder getStruct(final String value, final Qualifier.Builder qualifier) {

		return StructuredProperty.newBuilder().setValue(value).setQualifier(qualifier);

	}

	/*

	 * protected static StringField.Builder sf(final String s) { return StringField.newBuilder().setValue(s); }

	 * 

	 * protected static Qualifier.Builder getQualifier(final String classname, final String schemename) { return

	 * Qualifier.newBuilder().setClassid(classname).setClassname(classname).setSchemeid(schemename).setSchemename(schemename); }

	 */

}

package eu.dnetlib.pace.model;

import com.google.common.collect.Iterables;

import com.google.common.collect.Sets;

import com.google.common.collect.Sets.SetView;

import com.googlecode.protobuf.format.JsonFormat;

import eu.dnetlib.data.proto.FieldTypeProtos.Qualifier;

import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty;

import eu.dnetlib.pace.AbstractProtoPaceTest;

import eu.dnetlib.pace.config.Config;

import eu.dnetlib.pace.model.adaptor.Pid;

import org.junit.Test;

import static org.junit.Assert.assertFalse;

import static org.junit.Assert.assertTrue;

public class ProtoDocumentBuilderTest extends AbstractProtoPaceTest {

	@Test

	public void test_serialise1() {

		final String id = "12345";

		final Config config = getResultFullConf();

		final MapDocument document = ProtoDocumentBuilder.newInstance(id, getResult(id), config.model());

		assertFalse(document.fieldNames().isEmpty());

		assertFalse(Iterables.isEmpty(document.fields()));

		System.out.println("original:\n" + document);

		final String stringDoc = MapDocumentSerializer.toString(document);

		System.out.println("srialization:\n" + stringDoc);

		final MapDocument decoded = MapDocumentSerializer.decode(stringDoc.getBytes());

		final SetView<String> diff = Sets.difference(document.fieldNames(), decoded.fieldNames());

		assertTrue(diff.isEmpty());

		System.out.println("decoded:\n" + decoded);

	}

	@Test

	public void test_serialise2() {

		final String id = "12345";

		final String path = "/eu/dnetlib/pace/model/gt.author.manghi1.json";

		final Config config = getPersonConf();

		final MapDocument document = ProtoDocumentBuilder.newInstance(id, getPersonGT(path).getEntity(), config.model());

		assertFalse(document.fieldNames().isEmpty());

		assertFalse(Iterables.isEmpty(document.fields()));

		System.out.println("original:\n" + document);

		final String stringDoc = MapDocumentSerializer.toString(document);

		System.out.println("srialization:\n" + stringDoc);

		final MapDocument decoded = MapDocumentSerializer.decode(stringDoc.getBytes());

		final SetView<String> diff = Sets.difference(document.fieldNames(), decoded.fieldNames());

		assertTrue(diff.isEmpty());

		System.out.println("decoded:\n" + decoded);

	}

	@Test

	public void testPidSerialization() {

		final StructuredProperty sp = StructuredProperty.newBuilder().setValue("1234").setQualifier(

				Qualifier.newBuilder().setClassid("doi").setClassname("doi").setSchemeid("dnet:pid_types").setSchemename("dnet:pid_types")).build();

		final String json = JsonFormat.printToString(sp);

		final Pid pid = Pid.fromOafJson(json);

	}

}

package eu.dnetlib.pace.model.gt;

import com.google.common.collect.Sets;

import org.junit.Test;

import java.util.Set;

import static org.junit.Assert.assertTrue;

public class AuthorTest {

	@Test

	public void test() {

		final Set<Author> s1 = getAuthors(3);

		final Set<Author> s2 = getAuthors(3);

		final Set<Author> i = Sets.intersection(s1, s2);

		System.out.println(i);

		assertTrue(i.size() == 3);

	}

	@Test

	public void test1() {

		final Authors a1 = new Authors(a("1", "Wang, M."));

		final Authors a2 = new Authors(a("1", "Wang, M."));

		final Set<Author> i = Sets.intersection(a1, a2);

		assertTrue(i.size() == 1);

	}

	private Set<Author> getAuthors(final int n) {

		final Set<Author> s = Sets.newHashSet();

		for (int i = 0; i < n; i++) {

			s.add(a(i + "", "name" + i));

		}

		return s;

	}

	private Author a(final String id, final String fullname) {

		final Author a = new Author();

		a.setId(id);

		a.setFullname(fullname);

		return a;

	}

}

package eu.dnetlib.data.transform.xml;

import java.util.List;

import eu.dnetlib.data.transform.xml.OpenTrialsXsltFunctions.JsonProv;

import org.junit.After;

import org.junit.Before;

import org.junit.Test;

import static org.junit.Assert.assertEquals;

/**

 * OpenTrialsXsltFunctions Tester.

 *

 */

public class OpenTrialsXsltFunctionsTest {

	private String jsonProv = "[{\"url\" : \"http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT00378508\", \"sourceID\" : \"77eb42c5-0ec7-4e31-963a-5736b66f2d26\", \"sourceName\" : \"ictrp\"},{\"url\" : \"https://www.clinicaltrials.gov/ct2/show/NCT00378508?lup_e=02%2F04%2F2016&lup_s=01%2F01%2F2001&rank=175075&resultsxml=true\", \"sourceID\" : \"b389497c-0833-432b-a09b-930526b7b4d4\", \"sourceName\" : \"nct\"}]";

	private String jsonProvWithNull = "[{\"url\" : \"\", \"sourceID\" : \"77eb42c5-0ec7-4e31-963a-5736b66f2d26\", \"sourceName\" : \"ictrp\"},{\"url\" : \"https://www.clinicaltrials.gov/ct2/show/NCT00378508?lup_e=02%2F04%2F2016&lup_s=01%2F01%2F2001&rank=175075&resultsxml=true\", \"sourceID\" : \"b389497c-0833-432b-a09b-930526b7b4d4\", \"sourceName\" : \"nct\"}]";

	private String jidentifiers = "{112683,NCT00920439}";

	@Before

	public void before() throws Exception {

	}

	@After

	public void after() throws Exception {

	}

	/**

	 * Method: getProvs(String jsonProvList)

	 */

	@Test

	public void testGetProvs() throws Exception {

		List<JsonProv> list = OpenTrialsXsltFunctions.getProvs(jsonProv);

		assertEquals(2, list.size());

	}

	/**

	 * Method: getMainIdentifierURL(String jsonProvList)

	 */

	@Test

	public void testGetMainIdentifierURL() throws Exception {

		String url = OpenTrialsXsltFunctions.getMainIdentifierURL(jsonProv);

		assertEquals( "http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT00378508", url );

		url = OpenTrialsXsltFunctions.getMainIdentifierURL(jsonProvWithNull);

		assertEquals("https://www.clinicaltrials.gov/ct2/show/NCT00378508?lup_e=02%2F04%2F2016&lup_s=01%2F01%2F2001&rank=175075&resultsxml=true", url);

	}

} 

package eu.dnetlib.data.transform;

import java.io.IOException;

import java.io.InputStream;

import org.apache.commons.lang3.StringUtils;

import org.junit.Assert;

import org.junit.Test;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertNotNull;

import static org.junit.Assert.assertTrue;

/**

 * Created by claudio on 12/12/2016.

 */

public class OntologyLoaderTest {

	private String basePath = "/eu/dnetlib/bootstrap/profiles/OntologyDSResources/OntologyDSResourceType/";

	@Test

	public void testLoadOntologyFromCp() {

		final InputStream i = getClass().getResourceAsStream(basePath + "publication_publication_relations.xml");

        Ontology o = OntologyLoader.loadOntologyFromCp(i);

        checkOntology(o);

        String providedBy = o.inverseOf("isSupplementedBy");

        assertEquals(providedBy, "isSupplementTo");

        String provides = o.inverseOf("isPartOf");

        assertEquals(provides, "hasPart");

    }

	@Test

	public void testLoadOntologiesFromCp() throws IOException {

		OntologyLoader.loadOntologiesFromCp().values().forEach(o -> checkOntology(o));

	}

	@Test

	public void testLoadOntologiesSerialization() throws IOException {

		final Ontologies o = OntologyLoader.loadOntologiesFromCp();

		assertNotNull(o);

		final String json = o.toJson(true);

		System.out.println(json);

		assertTrue(StringUtils.isNoneBlank(json));

		final Ontologies o1 = OntologyLoader.loadOntologies(json);

		assertNotNull(o1);

		o1.entrySet().forEach(e -> checkOntology(e.getValue()));

	}

	private void checkOntology(Ontology o) {

		Assert.assertNotNull(o);

		Assert.assertTrue(StringUtils.isNotBlank(o.getCode()));

		Assert.assertTrue(StringUtils.isNotBlank(o.getDescription()));

		Assert.assertNotNull(o.getTerms().values());

		o.getTerms().values().forEach(it -> {

			Assert.assertTrue(StringUtils.isNotBlank(it.getCode()));

			Assert.assertTrue(StringUtils.isNotBlank(it.getEncoding()));

			Assert.assertTrue(StringUtils.isNotBlank(it.getEnglishName()));

			Assert.assertTrue(StringUtils.isNotBlank(it.getNativeName()));

			Assert.assertTrue(StringUtils.isNotBlank(it.getInverseCode()));

			Assert.assertNotNull(o.getTerms().get(it.getInverseCode()));

		});

	}

}

package eu.dnetlib.data.transform;

import java.io.IOException;

import java.io.StringWriter;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.Arrays;

import java.util.List;

import com.google.protobuf.InvalidProtocolBufferException;

import com.googlecode.protobuf.format.JsonFormat;

import eu.dnetlib.data.mapreduce.util.DNGFTest;

import eu.dnetlib.data.proto.DNGFProtos.DNGF;

import eu.dnetlib.data.proto.DNGFProtos.DNGFEntity;

import eu.dnetlib.data.proto.KindProtos.Kind;

import org.apache.commons.codec.binary.Base64;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.common.SolrInputDocument;

import org.apache.solr.common.SolrInputField;

import org.dom4j.DocumentException;

import org.junit.Before;

import org.junit.Test;

import static org.junit.Assert.assertFalse;

import static org.junit.Assert.assertNotNull;

public class SolrProtoMapperTest {

	private static final Log log = LogFactory.getLog(SolrProtoMapper.class); // NOPMD by marko on 11/24/08 5:02 PM

	private static final String outFormat = new String("yyyy-MM-dd'T'hh:mm:ss'Z'");

	private final static List<String> dateFormats = Arrays.asList("yyyy-MM-dd'T'hh:mm:ss", "yyyy-MM-dd", "dd-MM-yyyy", "dd/MM/yyyy", "yyyy");

	private final static String dataset = "{\"kind\": \"entity\",\"entity\": {\"type\": \"dataset\",\"originalId\": [\"http://esipfed.org/ns/fedsearch/1.1/data#\",\"C1293870510-PODAAC\",\"C1293870494-PODAAC\",\"C1293870514-PODAAC\",\"C1293870523-PODAAC\",\"http://esipfed.org/ns/fedsearch/1.1/metadata#\"],\"collectedfrom\": [{\"key\": \"10|wds_________::827fac6a4d016d53b6874573cf37fc05\",\"value\": \"Common Metadata Repository (CMR)\"}],\"pid\": [{\"value\": \"C1293870510-PODAAC\",\"qualifier\": {\"classid\": \"cmrid\",\"classname\": \"cmrid\",\"schemeid\": \"dnet:pid_types\",\"schemename\": \"dnet:pid_types\"}},{\"value\": \"C1293870494-PODAAC\",\"qualifier\": {\"classid\": \"cmrid\",\"classname\": \"cmrid\",\"schemeid\": \"dnet:pid_types\",\"schemename\": \"dnet:pid_types\"}},{\"value\": \"C1293870514-PODAAC\",\"qualifier\": {\"classid\": \"cmrid\",\"classname\": \"cmrid\",\"schemeid\": \"dnet:pid_types\",\"schemename\": \"dnet:pid_types\"}},{\"value\": \"C1293870523-PODAAC\",\"qualifier\": {\"classid\": \"cmrid\",\"classname\": \"cmrid\",\"schemeid\": \"dnet:pid_types\",\"schemename\": \"dnet:pid_types\"}}],\"dateofcollection\": \"2016-08-29T14:39:03.816+02:00\",\"id\": \"60|dedup_wf_001::002a1f239e51f7b92df75b7f8902c117\",\"dateoftransformation\": \"\",\"dataset\": {\"metadata\": {\"title\": [{\"value\": \"PODAAC-SEABY-ANILT\",\"qualifier\": {\"classid\": \"main title\",\"classname\": \"main title\",\"schemeid\": \"dnet:dataCite_title\",\"schemename\": \"dnet:dataCite_title\"}}],\"relevantdate\": [{\"value\": \"2003-04-10T00:08:15.000Z\",\"qualifier\": {\"classid\": \"UNKNOWN\",\"classname\": \"UNKNOWN\",\"schemeid\": \"dnet:dataCite_date\",\"schemename\": \"dnet:dataCite_date\"}}],\"dateofacceptance\": {\"value\": \"\"},\"publisher\": {\"value\": \"PO.DAAC\"},\"resulttype\": {\"classid\": \"dataset\",\"classname\": \"dataset\",\"schemeid\": \"dnet:result_typologies\",\"schemename\": \"dnet:result_typologies\"},\"language\": {\"classid\": \"und\",\"classname\": \"Undetermined\",\"schemeid\": \"dnet:languages\",\"schemename\": \"dnet:languages\"},\"resourcetype\": {\"classid\": \"Dataset\",\"classname\": \"Dataset\",\"schemeid\": \"dnet:dataCite_resource\",\"schemename\": \"dnet:dataCite_resource\"},\"size\": {\"value\": \"\"},\"format\": [{\"value\": \"ECHO10\"},{\"value\": \"ECHO10\"},{\"value\": \"ECHO10\"},{\"value\": \"ECHO10\"}],\"version\": {\"value\": \"\"},\"lastmetadataupdate\": {\"value\": \"2012-03-22T03:07:08.129Z\"},\"description\": [{\"value\": \" SeaWinds on ADEOS-II Level 3 Sigma-0 Polar-Stereographic Local-Time-of-Day Browse Images of Antarctic Region \"}]},\"instance\": [{\"licence\": {\"classid\": \"CLOSED\",\"classname\": \"Closed Access\",\"schemeid\": \"dnet:access_modes\",\"schemename\": \"dnet:access_modes\"},\"instancetype\": {\"classid\": \"0021\",\"classname\": \"Dataset\",\"schemeid\": \"dnet:dataCite_resource\",\"schemename\": \"dnet:dataCite_resource\"},\"hostedby\": {\"key\": \"10|openaire____::55045bd2a65019fd8e6741a755395c8c\",\"value\": \"Unknown Repository\"},\"url\": [\"\"]},{\"licence\": {\"classid\": \"CLOSED\",\"classname\": \"Closed Access\",\"schemeid\": \"dnet:access_modes\",\"schemename\": \"dnet:access_modes\"},\"instancetype\": {\"classid\": \"0021\",\"classname\": \"Dataset\",\"schemeid\": \"dnet:dataCite_resource\",\"schemename\": \"dnet:dataCite_resource\"},\"hostedby\": {\"key\": \"10|openaire____::55045bd2a65019fd8e6741a755395c8c\",\"value\": \"Unknown Repository\"},\"url\": [\"\"]},{\"licence\": {\"classid\": \"CLOSED\",\"classname\": \"Closed Access\",\"schemeid\": \"dnet:access_modes\",\"schemename\": \"dnet:access_modes\"},\"instancetype\": {\"classid\": \"0021\",\"classname\": \"Dataset\",\"schemeid\": \"dnet:dataCite_resource\",\"schemename\": \"dnet:dataCite_resource\"},\"hostedby\": {\"key\": \"10|openaire____::55045bd2a65019fd8e6741a755395c8c\",\"value\": \"Unknown Repository\"},\"url\": [\"\"]},{\"licence\": {\"classid\": \"CLOSED\",\"classname\": \"Closed Access\",\"schemeid\": \"dnet:access_modes\",\"schemename\": \"dnet:access_modes\"},\"instancetype\": {\"classid\": \"0021\",\"classname\": \"Dataset\",\"schemeid\": \"dnet:dataCite_resource\",\"schemename\": \"dnet:dataCite_resource\"},\"hostedby\": {\"key\": \"10|openaire____::55045bd2a65019fd8e6741a755395c8c\",\"value\": \"Unknown Repository\"},\"url\": [\"\"]}]},\"children\": [{\"type\": \"dataset\",\"originalId\": [\"C1293870514-PODAAC\",\"http://esipfed.org/ns/fedsearch/1.1/data#\",\"http://esipfed.org/ns/fedsearch/1.1/metadata#\"],\"collectedfrom\": [{\"key\": \"10|wds_________::827fac6a4d016d53b6874573cf37fc05\",\"value\": \"Common Metadata Repository (CMR)\"}],\"pid\": [{\"value\": \"C1293870514-PODAAC\",\"qualifier\": {\"classid\": \"cmrid\",\"classname\": \"cmrid\",\"schemeid\": \"dnet:pid_types\",\"schemename\": \"dnet:pid_types\"}}],\"dateofcollection\": \"2016-08-29T14:39:03.816+02:00\",\"id\": \"60|wds_cmr_____::ed18768335fcac40ae2cc062abc9e442\",\"dateoftransformation\": \"\",\"dataset\": {\"metadata\": {\"title\": [{\"value\": \"PODAAC-SEABY-ARBLT\",\"qualifier\": {\"classid\": \"main title\",\"classname\": \"main title\",\"schemeid\": \"dnet:dataCite_title\",\"schemename\": \"dnet:dataCite_title\"}}],\"relevantdate\": [{\"value\": \"2003-04-10T00:08:15.000Z\",\"qualifier\": {\"classid\": \"UNKNOWN\",\"classname\": \"UNKNOWN\",\"schemeid\": \"dnet:dataCite_date\",\"schemename\": \"dnet:dataCite_date\"}}],\"dateofacceptance\": {\"value\": \"\"},\"publisher\": {\"value\": \"PO.DAAC\"},\"resulttype\": {\"classid\": \"dataset\",\"classname\": \"dataset\",\"schemeid\": \"dnet:result_typologies\",\"schemename\": \"dnet:result_typologies\"},\"language\": {\"classid\": \"und\",\"classname\": \"Undetermined\",\"schemeid\": \"dnet:languages\",\"schemename\": \"dnet:languages\"},\"resourcetype\": {\"classid\": \"Dataset\",\"classname\": \"Dataset\",\"schemeid\": \"dnet:dataCite_resource\",\"schemename\": \"dnet:dataCite_resource\"},\"size\": {\"value\": \"\"},\"format\": [{\"value\": \"ECHO10\"}],\"version\": {\"value\": \"\"},\"lastmetadataupdate\": {\"value\": \"2012-03-22T01:56:49.933Z\"},\"description\": [{\"value\": \" SeaWinds on ADEOS-II Level 3 Sigma-0 Polar-Stereographic Local-Time-of-Day Browse Maps of Arctic Region \"}]},\"instance\": [{\"licence\": {\"classid\": \"CLOSED\",\"classname\": \"Closed Access\",\"schemeid\": \"dnet:access_modes\",\"schemename\": \"dnet:access_modes\"},\"instancetype\": {\"classid\": \"0021\",\"classname\": \"Dataset\",\"schemeid\": \"dnet:dataCite_resource\",\"schemename\": \"dnet:dataCite_resource\"},\"hostedby\": {\"key\": \"10|openaire____::55045bd2a65019fd8e6741a755395c8c\",\"value\": \"Unknown Repository\"},\"url\": [\"\"]}]},\"oaiprovenance\": {\"originDescription\": {\"harvestDate\": \"2016-08-29T14:39:03.816+02:00\",\"altered\": true,\"baseURL\": \"https%3A%2F%2Fcmr.earthdata.nasa.gov%2Fsearch%2Fcollections.json%3Fpretty%3Dtrue\",\"identifier\": \"\",\"datestamp\": \"\",\"metadataNamespace\": \"\"}}},{\"type\": \"dataset\",\"originalId\": [\"C1293870494-PODAAC\",\"http://esipfed.org/ns/fedsearch/1.1/data#\",\"http://esipfed.org/ns/fedsearch/1.1/metadata#\"],\"collectedfrom\": [{\"key\": \"10|wds_________::827fac6a4d016d53b6874573cf37fc05\",\"value\": \"Common Metadata Repository (CMR)\"}],\"pid\": [{\"value\": \"C1293870494-PODAAC\",\"qualifier\": {\"classid\": \"cmrid\",\"classname\": \"cmrid\",\"schemeid\": \"dnet:pid_types\",\"schemename\": \"dnet:pid_types\"}}],\"dateofcollection\": \"2016-08-29T14:39:03.816+02:00\",\"id\": \"60|wds_cmr_____::ac6f36f8c9ef39f815045eaa1182e745\",\"dateoftransformation\": \"\",\"dataset\": {\"metadata\": {\"title\": [{\"value\": \"PODAAC-SEABY-ARILT\",\"qualifier\": {\"classid\": \"main title\",\"classname\": \"main title\",\"schemeid\": \"dnet:dataCite_title\",\"schemename\": \"dnet:dataCite_title\"}}],\"relevantdate\": [{\"value\": \"2003-04-10T00:08:15.000Z\",\"qualifier\": {\"classid\": \"UNKNOWN\",\"classname\": \"UNKNOWN\",\"schemeid\": \"dnet:dataCite_date\",\"schemename\": \"dnet:dataCite_date\"}}],\"dateofacceptance\": {\"value\": \"\"},\"publisher\": {\"value\": \"PO.DAAC\"},\"resulttype\": {\"classid\": \"dataset\",\"classname\": \"dataset\",\"schemeid\": \"dnet:result_typologies\",\"schemename\": \"dnet:result_typologies\"},\"language\": {\"classid\": \"und\",\"classname\": \"Undetermined\",\"schemeid\": \"dnet:languages\",\"schemename\": \"dnet:languages\"},\"resourcetype\": {\"classid\": \"Dataset\",\"classname\": \"Dataset\",\"schemeid\": \"dnet:dataCite_resource\",\"schemename\": \"dnet:dataCite_resource\"},\"size\": {\"value\": \"\"},\"format\": [{\"value\": \"ECHO10\"}],\"version\": {\"value\": \"\"},\"lastmetadataupdate\": {\"value\": \"2012-03-22T02:51:51.788Z\"},\"description\": [{\"value\": \" SeaWinds on ADEOS-II Level 3 Sigma-0 Polar-Stereographic Local-Time-of-Day Browse Images of Arctic Region \"}]},\"instance\": [{\"licence\": {\"classid\": \"CLOSED\",\"classname\": \"Closed Access\",\"schemeid\": \"dnet:access_modes\",\"schemename\": \"dnet:access_modes\"},\"instancetype\": {\"classid\": \"0021\",\"classname\": \"Dataset\",\"schemeid\": \"dnet:dataCite_resource\",\"schemename\": \"dnet:dataCite_resource\"},\"hostedby\": {\"key\": \"10|openaire____::55045bd2a65019fd8e6741a755395c8c\",\"value\": \"Unknown Repository\"},\"url\": [\"\"]}]},\"oaiprovenance\": {\"originDescription\": {\"harvestDate\": \"2016-08-29T14:39:03.816+02:00\",\"altered\": true,\"baseURL\": \"https%3A%2F%2Fcmr.earthdata.nasa.gov%2Fsearch%2Fcollections.json%3Fpretty%3Dtrue\",\"identifier\": \"\",\"datestamp\": \"\",\"metadataNamespace\": \"\"}}},{\"type\": \"dataset\",\"originalId\": [\"C1293870523-PODAAC\",\"http://esipfed.org/ns/fedsearch/1.1/data#\",\"http://esipfed.org/ns/fedsearch/1.1/metadata#\"],\"collectedfrom\": [{\"key\": \"10|wds_________::827fac6a4d016d53b6874573cf37fc05\",\"value\": \"Common Metadata Repository (CMR)\"}],\"pid\": [{\"value\": \"C1293870523-PODAAC\",\"qualifier\": {\"classid\": \"cmrid\",\"classname\": \"cmrid\",\"schemeid\": \"dnet:pid_types\",\"schemename\": \"dnet:pid_types\"}}],\"dateofcollection\": \"2016-08-29T14:39:03.816+02:00\",\"id\": \"60|wds_cmr_____::a26b222d236fd523df9711c21a879911\",\"dateoftransformation\": \"\",\"dataset\": {\"metadata\": {\"title\": [{\"value\": \"PODAAC-SEABY-ANBLT\",\"qualifier\": {\"classid\": \"main title\",\"classname\": \"main title\",\"schemeid\": \"dnet:dataCite_title\",\"schemename\": \"dnet:dataCite_title\"}}],\"relevantdate\": [{\"value\": \"2003-04-10T00:08:15.000Z\",\"qualifier\": {\"classid\": \"UNKNOWN\",\"classname\": \"UNKNOWN\",\"schemeid\": \"dnet:dataCite_date\",\"schemename\": \"dnet:dataCite_date\"}}],\"dateofacceptance\": {\"value\": \"\"},\"publisher\": {\"value\": \"PO.DAAC\"},\"resulttype\": {\"classid\": \"dataset\",\"classname\": \"dataset\",\"schemeid\": \"dnet:result_typologies\",\"schemename\": \"dnet:result_typologies\"},\"language\": {\"classid\": \"und\",\"classname\": \"Undetermined\",\"schemeid\": \"dnet:languages\",\"schemename\": \"dnet:languages\"},\"resourcetype\": {\"classid\": \"Dataset\",\"classname\": \"Dataset\",\"schemeid\": \"dnet:dataCite_resource\",\"schemename\": \"dnet:dataCite_resource\"},\"size\": {\"value\": \"\"},\"format\": [{\"value\": \"ECHO10\"}],\"version\": {\"value\": \"\"},\"lastmetadataupdate\": {\"value\": \"2012-03-22T03:01:46.198Z\"},\"description\": [{\"value\": \" SeaWinds on ADEOS-II Level 3 Sigma-0 Polar-Stereographic Local-Time-of-Day Browse Maps of Antarctic Region \"}]},\"instance\": [{\"licence\": {\"classid\": \"CLOSED\",\"classname\": \"Closed Access\",\"schemeid\": \"dnet:access_modes\",\"schemename\": \"dnet:access_modes\"},\"instancetype\": {\"classid\": \"0021\",\"classname\": \"Dataset\",\"schemeid\": \"dnet:dataCite_resource\",\"schemename\": \"dnet:dataCite_resource\"},\"hostedby\": {\"key\": \"10|openaire____::55045bd2a65019fd8e6741a755395c8c\",\"value\": \"Unknown Repository\"},\"url\": [\"\"]}]},\"oaiprovenance\": {\"originDescription\": {\"harvestDate\": \"2016-08-29T14:39:03.816+02:00\",\"altered\": true,\"baseURL\": \"https%3A%2F%2Fcmr.earthdata.nasa.gov%2Fsearch%2Fcollections.json%3Fpretty%3Dtrue\",\"identifier\": \"\",\"datestamp\": \"\",\"metadataNamespace\": \"\"}}},{\"type\": \"dataset\",\"originalId\": [\"C1293870510-PODAAC\",\"http://esipfed.org/ns/fedsearch/1.1/data#\",\"http://esipfed.org/ns/fedsearch/1.1/metadata#\"],\"collectedfrom\": [{\"key\": \"10|wds_________::827fac6a4d016d53b6874573cf37fc05\",\"value\": \"Common Metadata Repository (CMR)\"}],\"pid\": [{\"value\": \"C1293870510-PODAAC\",\"qualifier\": {\"classid\": \"cmrid\",\"classname\": \"cmrid\",\"schemeid\": \"dnet:pid_types\",\"schemename\": \"dnet:pid_types\"}}],\"dateofcollection\": \"2016-08-29T14:39:03.816+02:00\",\"id\": \"60|wds_cmr_____::002a1f239e51f7b92df75b7f8902c117\",\"dateoftransformation\": \"\",\"dataset\": {\"metadata\": {\"title\": [{\"value\": \"PODAAC-SEABY-ANILT\",\"qualifier\": {\"classid\": \"main title\",\"classname\": \"main title\",\"schemeid\": \"dnet:dataCite_title\",\"schemename\": \"dnet:dataCite_title\"}}],\"relevantdate\": [{\"value\": \"2003-04-10T00:08:15.000Z\",\"qualifier\": {\"classid\": \"UNKNOWN\",\"classname\": \"UNKNOWN\",\"schemeid\": \"dnet:dataCite_date\",\"schemename\": \"dnet:dataCite_date\"}}],\"dateofacceptance\": {\"value\": \"\"},\"publisher\": {\"value\": \"PO.DAAC\"},\"resulttype\": {\"classid\": \"dataset\",\"classname\": \"dataset\",\"schemeid\": \"dnet:result_typologies\",\"schemename\": \"dnet:result_typologies\"},\"language\": {\"classid\": \"und\",\"classname\": \"Undetermined\",\"schemeid\": \"dnet:languages\",\"schemename\": \"dnet:languages\"},\"resourcetype\": {\"classid\": \"Dataset\",\"classname\": \"Dataset\",\"schemeid\": \"dnet:dataCite_resource\",\"schemename\": \"dnet:dataCite_resource\"},\"size\": {\"value\": \"\"},\"format\": [{\"value\": \"ECHO10\"}],\"version\": {\"value\": \"\"},\"lastmetadataupdate\": {\"value\": \"2012-03-22T03:07:08.129Z\"},\"description\": [{\"value\": \" SeaWinds on ADEOS-II Level 3 Sigma-0 Polar-Stereographic Local-Time-of-Day Browse Images of Antarctic Region \"}]},\"instance\": [{\"licence\": {\"classid\": \"CLOSED\",\"classname\": \"Closed Access\",\"schemeid\": \"dnet:access_modes\",\"schemename\": \"dnet:access_modes\"},\"instancetype\": {\"classid\": \"0021\",\"classname\": \"Dataset\",\"schemeid\": \"dnet:dataCite_resource\",\"schemename\": \"dnet:dataCite_resource\"},\"hostedby\": {\"key\": \"10|openaire____::55045bd2a65019fd8e6741a755395c8c\",\"value\": \"Unknown Repository\"},\"url\": [\"\"]}]},\"oaiprovenance\": {\"originDescription\": {\"harvestDate\": \"2016-08-29T14:39:03.816+02:00\",\"altered\": true,\"baseURL\": \"https%3A%2F%2Fcmr.earthdata.nasa.gov%2Fsearch%2Fcollections.json%3Fpretty%3Dtrue\",\"identifier\": \"\",\"datestamp\": \"\",\"metadataNamespace\": \"\"}}}],\"oaiprovenance\": {\"originDescription\": {\"harvestDate\": \"2016-08-29T14:39:03.816+02:00\",\"altered\": true,\"baseURL\": \"https%3A%2F%2Fcmr.earthdata.nasa.gov%2Fsearch%2Fcollections.json%3Fpretty%3Dtrue\",\"identifier\": \"\",\"datestamp\": \"\",\"metadataNamespace\": \"\"}}},\"dataInfo\": {\"inferred\": true,\"deletedbyinference\": false,\"trust\": \"0.9\",\"inferenceprovenance\": \"dedup-similarity-dataset\",\"provenanceaction\": {\"classid\": \"sysimport:dedup\",\"classname\": \"sysimport:dedup\",\"schemeid\": \"dnet:provenanceActions\",\"schemename\": \"dnet:provenanceActions\"}}}";

	private String fields;

	@Before

	public void setUp() throws IOException {

		final StringWriter sw = new StringWriter();

		IOUtils.copy(getClass().getResourceAsStream("fields.xml"), sw);

		fields = sw.toString();

		assertNotNull(fields);

		assertFalse(fields.isEmpty());

		log.info(fields);

	}

	@Test

	public void testProto2SolrDocument() throws DocumentException, InvalidProtocolBufferException, JsonFormat.ParseException {

		final SolrProtoMapper mapper = new SolrProtoMapper(fields);

		assertNotNull(mapper);

		DNGF.Builder builder = DNGF.newBuilder();

		JsonFormat.merge(dataset, builder);

		final DNGF oaf = builder.build();

		assertNotNull(oaf.getEntity().getChildrenList());

		assertFalse(oaf.getEntity().getChildrenList().isEmpty());

		log.info("byte[] size: " + oaf.toByteArray().length);

		log.info("json size:   " + JsonFormat.printToString(oaf).length());

		log.info("base64 size: " + Base64.encodeBase64String(oaf.toByteArray()).length());

		final byte[] decodeBase64 = Base64.decodeBase64(Base64.encodeBase64String(oaf.toByteArray()));

		log.info("decoded: " + JsonFormat.printToString(DNGF.parseFrom(decodeBase64)));

		final SolrInputDocument doc = mapper.map(oaf, getParsedDateField("2015-02-15"), "asd", "action-set");

		assertNotNull(doc);

		for (final SolrInputField f : doc.values()) {

			log.info(f);

		}

	}

	/**

	 * method return a solr-compatible string representation of a date

	 *

	 * @param date

	 * @return

	 * @throws DocumentException

	 * @throws ParseException

	 */

	public String getParsedDateField(final String date) {

		for (String formatString : dateFormats) {

			try {

				return new SimpleDateFormat(outFormat).format(new SimpleDateFormat(formatString).parse(date));

			} catch (ParseException e) {}

		}

		throw new IllegalStateException("unable to parse date: " + date);

	}

}

package eu.dnetlib.data.graph.utils;

import org.junit.Test;

import static org.junit.Assert.assertEquals;

/**

 * Created by claudio on 02/02/2017.

 */

public class RelDescriptorTest {

	@Test

	public void testRelDesriptor() {

		final RelDescriptor rd = new RelDescriptor("publication_publication_isMergedIn");

		assertEquals("publication_publication", rd.getOntologyCode());

		assertEquals("publication_publication", rd.asQualifier().getSchemeid());

		assertEquals("isMergedIn", rd.getTermCode());

		assertEquals("isMergedIn", rd.asQualifier().getClassid());

		//assertEquals(rd.getTargetId(), "50|dedup_wf_001::01d734887f7e33fc754b22de0940c4ab");

	}

	@Test

	public void testRelDesriptorFull() {

		final RelDescriptor rd = new RelDescriptor("publication_publication_isMergedIn:50|dedup_wf_001::01d734887f7e33fc754b22de0940c4ab");

		assertEquals("publication_publication", rd.getOntologyCode());

		assertEquals("publication_publication", rd.asQualifier().getSchemeid());

		assertEquals("isMergedIn", rd.getTermCode());

		assertEquals("isMergedIn", rd.asQualifier().getClassid());

		assertEquals("50|dedup_wf_001::01d734887f7e33fc754b22de0940c4ab", rd.getTargetId());

	}

	@Test(expected = IllegalArgumentException.class)

	public void testRelDesriptorMalformed() {

		new RelDescriptor("asd");

	}

	@Test(expected = IllegalArgumentException.class)

	public void testRelDesriptorEmpty() {

		new RelDescriptor("");

	}

}

package eu.dnetlib.data.mapreduce.util;

import com.google.protobuf.GeneratedMessage;

import com.google.protobuf.InvalidProtocolBufferException;

import eu.dnetlib.data.graph.model.DNGFDecoder;

import eu.dnetlib.data.proto.DNGFProtos.DNGF;

import eu.dnetlib.data.proto.DNGFProtos.DNGFEntity;

import eu.dnetlib.data.proto.DNGFProtos.DNGFRel;

import eu.dnetlib.data.proto.DatasourceProtos.Datasource;

import eu.dnetlib.data.proto.FieldTypeProtos.*;

import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty.Builder;

import eu.dnetlib.data.proto.KindProtos.Kind;

import eu.dnetlib.data.proto.OrganizationProtos.Organization;

import eu.dnetlib.data.proto.PersonProtos.Person;

import eu.dnetlib.data.proto.ProjectProtos.Project;

import eu.dnetlib.data.proto.PublicationProtos.Publication;

import eu.dnetlib.data.proto.TypeProtos.Type;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

public class DNGFTest {

	public static final String CITATION_JSON =

			"<citations>\n  <citation>\n    <rawText>[10] M. Foret et al., Phys. Rev. B 66, 024204 (2002).</rawText>\n  </citation>\n  <citation>\n    <rawText>[11] B. Ru\175404\264e et al., Phys. Rev. Lett. 90, 095502 (2003).</rawText>\n  </citation>\n  <citation>\n    <rawText>[12] U. Buchenau et al., Phys. Rev. B 34, 5665 (1986).</rawText>\n  </citation>\n  <citation>\n    <rawText>[13] S.N. Taraskin and S.R. Elliott, J. Phys.: Condens. Mat- ter 11, A219 (1999).</rawText>\n  </citation>\n  <citation>\n    <rawText>[14] B. Hehlen et al., Phys. Rev. Lett. 84, 5355 (2000).</rawText>\n  </citation>\n  <citation>\n    <rawText>[15] N.V. Surotsev et al., J. Phys.: Condens. Matter 10, L113 (1998).</rawText>\n  </citation>\n  <citation>\n    <rawText>[16] D.A. Parshin and C. Laermans, Phys. Rev. B 63, 132203 (2001).</rawText>\n  </citation>\n  <citation>\n    <rawText>[17] V.L. Gurevich et al., Phys. Rev. B 67, 094203 (2003).</rawText>\n  </citation>\n  <citation>\n    <rawText>[18] A. Matic et al., Phys. Rev. Lett. 86, 3803 (2001).</rawText>\n  </citation>\n  <citation>\n    <rawText>[19] E. Rat et al., arXiv:cond-mat/0505558, 23 May 2005.</rawText>\n  </citation>\n  <citation>\n    <rawText>[1] R.C. Zeller and R.O. Pohl, Phys. Rev. B 4, 2029 (1971).</rawText>\n  </citation>\n  <citation>\n    <rawText>[20] C.A. Angell, J. Non-Cryst. Solids 131\20023133, 13 (1991).</rawText>\n  </citation>\n  <citation>\n    <rawText>[21] A.P. Sokolov et al., Phys. Rev. Lett. 71, 2062 (1993).</rawText>\n  </citation>\n  <citation>\n    <rawText>[22] T. Matsuo et al., Solid State Ionics 154-155, 759 (2002).</rawText>\n  </citation>\n  <citation>\n    <rawText>[23] V.K. Malinovsky et al., Europhys. Lett. 11, 43 (1990).</rawText>\n  </citation>\n  <citation>\n    <rawText>[24] J. Lor\250osch et al., J. Non-Cryst. Solids 69, 1 (1984).</rawText>\n  </citation>\n  <citation>\n    <rawText>[25] U. Buchenau, Z. Phys. B 58, 181 (1985).</rawText>\n  </citation>\n  <citation>\n    <rawText>[26] A.F. Io\175400e and A.R. Regel, Prog. Semicond. 4, 237 (1960).</rawText>\n  </citation>\n  <citation>\n    <rawText>[27] R. Dell\20031Anna et al., Phys. Rev. Lett. 80, 1236 (1998).</rawText>\n  </citation>\n  <citation>\n    <rawText>[28] D. Fioretto et al., Phys. Rev. E 59, 4470 (1999).</rawText>\n  </citation>\n  <citation>\n    <rawText>[29] U. Buchenau et al., Phys. Rev. Lett. 77, 4035 (1996).</rawText>\n  </citation>\n  <citation>\n    <rawText>[2] M. Rothenfusser et al., Phys. Rev. B 27, 5196 (1983).</rawText>\n  </citation>\n  <citation>\n    <rawText>[30] J. Mattsson et al., J. Phys.: Condens. Matter 15, S1259 (2003).</rawText>\n  </citation>\n  <citation>\n    <rawText>[31] T. Scopigno et al., Phys. Rev. Lett. 92, 025503 (2004).</rawText>\n  </citation>\n  <citation>\n    <rawText>[32] M. Foret et al., Phys. Rev. Lett. 81, 2100 (1998).</rawText>\n  </citation>\n  <citation>\n    <rawText>[33] F. Sette et al., Science 280, 1550 (1998).</rawText>\n  </citation>\n  <citation>\n    <rawText>[34] J. Wuttke et al., Phys. Rev. E 52, 4026 (1995).</rawText>\n  </citation>\n  <citation>\n    <rawText>[35] M.A. Ramos et al., Phys. Rev. Lett. 78, 82 (1997).</rawText>\n  </citation>\n  <citation>\n    <rawText>[36] G. Monaco et al., Phys. Rev. Lett. 80, 2161 (1998).</rawText>\n  </citation>\n  <citation>\n    <rawText>[37] A. T\250olle, Rep. Prog. Phys. 64, 1473 (2001).</rawText>\n  </citation>\n  <citation>\n    <rawText>[38] As the straight lines do not cross the origin, this does not 2 imply \1623 \21035 \1651 .</rawText>\n  </citation>\n  <citation>\n    <rawText>[39] A. Matic et al., Europhys. Lett. 54, 77 (2001).</rawText>\n  </citation>\n  <citation>\n    <rawText>[3] S. Hunklinger and W. Arnold, in Physical Acoustics, Vol. XII, W.P. Mason and R.N. Thurston Eds. (Academic Press, N.Y. 1976), p. 155.</rawText>\n  </citation>\n  <citation>\n    <rawText>[40] IXS data are usually not available below \1651co, mostly for experimental reasons. E.g., that the rapid onset was not evidenced in vitreous silica [27], is not indicative of its absence but rather of a low qco \21074 1 nm\210221.</rawText>\n  </citation>\n  <citation>\n    <rawText>[41] G. Ruocco et al., Phys. Rev. Lett. 83, 5583 (1999).</rawText>\n  </citation>\n  <citation>\n    <rawText>[42] D. C\1307 iplys et al., J. Physique (Paris) 42, C6-184 (1981).</rawText>\n  </citation>\n  <citation>\n    <rawText>[43] R. Vacher et al., Rev. Sci. Instrum. 51, 288 (1980).</rawText>\n  </citation>\n  <citation>\n    <rawText>[44] R. Vacher et al., arXiv:cond-mat/0505560, 23 May 2005.</rawText>\n  </citation>\n  <citation>\n    <rawText>[45] T.N. Claytor et al., Phys. Rev. B 18, 5842 (1978).</rawText>\n  </citation>\n  <citation>\n    <rawText>[46] M. Arai et al., Physica B 263-264, 268 (1999).</rawText>\n  </citation>\n  <citation>\n    <rawText>[4] R. Vacher et al., J. Non-Cryst. Solids 45, 397 (1981); T.C. Zhu et al., Phys. Rev. B 44, 4281 (1991).</rawText>\n  </citation>\n  <citation>\n    <rawText>[5] J.E. Graebner et al., Phys. Rev. B 34, 5696 (1986).</rawText>\n  </citation>\n  <citation>\n    <rawText>[6] E. Duval and A. Mermet, Phys. Rev. B 58, 8159 (1998).</rawText>\n  </citation>\n  <citation>\n    <rawText>[7] A. Matic et al., Phys. Rev. Lett. 93, 145502 (2004).</rawText>\n  </citation>\n  <citation>\n    <rawText>[8] Often alluded to, e.g. in the Encyclopedia of Materials: Science and Technology, K.H.J. Buschow et al., Eds., Vol. 1 (Elsevier, Oxford, 2001), articles by S.R. Elliott on pp. 171-174 and U. Buchenau on pp. 212-215.</rawText>\n  </citation>\n  <citation>\n    <rawText>[9] E. Rat et al., Phys. Rev. Lett. 83, 1355 (1999).</rawText>\n  </citation>\n</citations>";

	public static final String STATISTICS_JSON =

			"[{ \"citationsPerYear\": \"many\", \"anotherCoolStatistic\": \"WoW\", \"nestedStat\": { \"firstNestedStat\" : \"value 1\", \"secondNestedStat\" : \"value 2\"}, \"listingStat\" : [ \"one\", \"two\" ] }]";

    private static final Log log = LogFactory.getLog(DNGFTest.class);

    private static String basePathProfiles = "/eu/dnetlib/test/profiles/OntologyDSResources/OntologyDSResourceType";

	public static StructuredProperty.Builder getStructuredproperty(final String value, final String classname, final String schemename) {

		return getStructuredproperty(value, classname, schemename, null);

	}

	public static StructuredProperty.Builder getStructuredproperty(final String value, final String classname, final String schemename, final DataInfo dataInfo) {

		final Builder sp = StructuredProperty.newBuilder().setValue(value).setQualifier(getQualifier(classname, schemename));

		if (dataInfo != null) {

			sp.setDataInfo(dataInfo);

		}

		return sp;

	}

	public static Qualifier.Builder getQualifier(final String classname, final String schemename) {

		return Qualifier.newBuilder().setClassid(classname).setClassname(classname).setSchemeid(schemename).setSchemename(schemename);

	}

	public static KeyValue getKV(final String id, final String name) {

		return KeyValue.newBuilder().setKey(id).setValue(name).build();

	}

	public static DNGFEntity getDatasource(final String datasourceId) {

		return DNGFEntity

				.newBuilder()

				.setType(Type.datasource)

				.setId(datasourceId)

				.setDatasource(

						Datasource.newBuilder().setMetadata(

								Datasource.Metadata.newBuilder().setOfficialname(sf("officialname")).setEnglishname(sf("englishname"))

										.setWebsiteurl(sf("websiteurl")).setContactemail(sf("contactemail")).addAccessinfopackage(sf("accessinforpackage"))

										.setNamespaceprefix(sf("namespaceprofix")).setDescription(sf("description")).setOdnumberofitems(sf("numberofitems"))

										.setOdnumberofitemsdate(sf("numberofitems date"))

										// .addOdsubjects("subjects")

										.setOdpolicies(sf("policies")).addOdlanguages(sf("languages")).addOdcontenttypes(sf("contenttypes"))

										.setDatasourcetype(getQualifier("type class", "type scheme")))).build();

	}

	public static DNGFEntity getResult(final String id) {

		return getResultBuilder(id).build();

	}

	public static DNGFEntity.Builder getResultBuilder(final String id) {

		return DNGFEntity

				.newBuilder()

				.setType(Type.publication)

				.setId(id)

				.setPublication(

						Publication.newBuilder()

								.setMetadata(

										Publication.Metadata

												.newBuilder()

												.addTitle(

														getStructuredproperty(

																"Analysis of cell viability in intervertebral disc: Effect of endplate permeability on cell population",

																"main title", "dnet:result_titles", getDataInfo()))

												.addTitle(getStructuredproperty("Another title", "alternative title", "dnet:result_titles", getDataInfo()))

												.addSubject(getStructuredproperty("Biophysics", "subject", "dnet:result_sujects"))

												.setDateofacceptance(sf("2010-01-01")).addSource(sf("sourceA")).addSource(sf("sourceB"))

												.addContext(Context.newBuilder().setId("egi::virtual::970"))

												.addContext(Context.newBuilder().setId("egi::classification::natsc::math::applied"))

												.addContext(Context.newBuilder().setId("egi::classification::natsc::math"))

												.addContext(Context.newBuilder().setId("egi::classification::natsc"))

												.addContext(Context.newBuilder().setId("egi::classification")).addContext(Context.newBuilder().setId("egi"))

												.addDescription(sf("Responsible for making and maintaining the extracellular matrix ..."))

												.addDescription(sf("Another description ...")).setPublisher(sf("ELSEVIER SCI LTD"))

												.setResulttype(getQualifier("publication", "dnet:result_types"))

												.setLanguage(getQualifier("eng", "dnet:languages"))).addInstance(getInstance("10|od__10", "Uk pubmed"))

								.addInstance(getInstance("10|od__10", "arxiv")))

				.addCollectedfrom(getKV("opendoar____::1064", "Oxford University Research Archive"))

				.addPid(getStructuredproperty("doi:74293", "doi", "dnet:pids")).addPid(getStructuredproperty("oai:74295", "oai", "dnet:pids"))

				.setDateofcollection("");

	}

	public static DataInfo getDataInfo() {

		return getDataInfo("0.4");

	}

	public static DataInfo getDataInfo(final String trust) {

		return DataInfo.newBuilder().setDeletedbyinference(false).setTrust("0.4").setInferenceprovenance("algo").setProvenanceaction(getQualifier("xx", "yy"))

				.build();

	}

	public static Instance.Builder getInstance(final String hostedbyId, final String hostedbyName) {

		return Instance.newBuilder().setHostedby(getKV(hostedbyId, hostedbyName)).setLicence(getQualifier("OpenAccess", "dnet:access_modes"))

				.setInstancetype(getQualifier("publication", "dnet:result_typologies")).addUrl("webresource url");

	}

	public static DNGFRel getDedupRel(final String source, final String target) {

		return DNGFRel.newBuilder().setSource(source).setTarget(target).setRelType(getQualifier("dedup", "dedupScheme"))

				.setChild(false).setCachedTarget(getResult(target)).build();

	}

	public static DNGFEntity getPerson() {

		return DNGFEntity

				.newBuilder()

				.setType(Type.person)

				.setId("WOS:000277866500014_A._Shirazi-Adl")

				.setPerson(

						Person.newBuilder().setMetadata(

								Person.Metadata.newBuilder().addSecondnames(sf("Shirazi-Adl")).setFullname(sf("A. Shirazi-Adl"))

										.setEmail(sf("name.surname@gmail.com")).setPhone(sf("12345")).setNationality(getQualifier("EN", "dnet:countries"))))

				.build();

	}

	public static DNGFEntity getOrganization(final String orgId) {

		return DNGFEntity

				.newBuilder()

				.setType(Type.organization)

				.setId(orgId)

				.addCollectedfrom(getKV("opendoar_1234", "UK pubmed"))

				.setOrganization(

						Organization.newBuilder().setMetadata(

								Organization.Metadata.newBuilder().setLegalname(sf("CENTRE D'APPUI A LA RECHERCHE ET A LA FORMATION GIE"))

										.setLegalshortname(sf("CAREF")).setWebsiteurl(sf("www.caref-mali.org"))

										.setCountry(getQualifier("ML", "dnet:countries")))).build();

	}

	public static DNGFEntity getProjectFP7(final String projectId, final String fundingProgram) throws InvalidProtocolBufferException {

		return DNGFEntity

				.newBuilder()

				.setType(Type.project)

				.setId(projectId)

				.addCollectedfrom(getKV("opendoar_1234", "UK pubmed"))

				.setProject(

						Project.newBuilder()

								.setMetadata(

										Project.Metadata

												.newBuilder()

												.setAcronym(sf("5CYRQOL"))

												.setTitle(sf("Cypriot Researchers Contribute to our Quality of Life"))

												.setStartdate(sf("2007-05-01"))

												.setEnddate(sf("2007-10-31"))

												.setEcsc39(sf("false"))

												.setContracttype(getQualifier("CSA", "ec:FP7contractTypes"))

												.addFundingtree(

														sf("<fundingtree><funder><id>ec__________::EC</id><shortname>EC</shortname><name>European Commission</name></funder><funding_level_2><id>ec__________::EC::FP7::"

																+ fundingProgram

																+ "::PEOPLE</id><description>Marie-Curie Actions</description><name>PEOPLE</name><class>ec:program</class><parent><funding_level_1><id>ec__________::EC::FP7::"

																+ fundingProgram

																+ "</id><description>"

																+ fundingProgram

																+ "-People</description><name>"

																+ fundingProgram

																+ "</name><class>ec:specificprogram</class><parent><funding_level_0><id>ec__________::EC::FP7</id><description>SEVENTH FRAMEWORK PROGRAMME</description><name>FP7</name><parent/><class>ec:frameworkprogram</class></funding_level_0></parent></funding_level_1></parent></funding_level_2></fundingtree>"))))

				.build();

	}

	public static DNGFEntity getProjectWT() throws InvalidProtocolBufferException {

		return DNGFEntity

				.newBuilder()

				.setType(Type.project)

				.setId("project|wt::087536")

				.addCollectedfrom(getKV("wellcomeTrust", "wellcome trust"))

				.setProject(

						Project.newBuilder()

								.setMetadata(

										Project.Metadata

												.newBuilder()

												.setAcronym(sf("UNKNOWN"))

												.setTitle(sf("Research Institute for Infectious Diseases of Poverty (IIDP)."))

												.setStartdate(sf("2007-05-01"))

												.setEnddate(sf("2007-10-31"))

												.setEcsc39(sf("false"))

												.setContracttype(getQualifier("UNKNOWN", "wt:contractTypes"))

												.addFundingtree(

														sf("<fundingtree><funder><id>wt__________::WT</id><shortname>WT</shortname><name>Wellcome Trust</name></funder><funding_level_0><id>wt__________::WT::UNKNOWN</id><description>UNKNOWN</description><name>UNKNOWN</name><class>wt:fundingStream</class><parent/></funding_level_0></fundingtree>"))

												.addFundingtree(

														sf("<fundingtree><funder><id>wt__________::WT</id><shortname>WT</shortname><name>Wellcome Trust</name></funder><funding_level_0><id>wt__________::WT::Technology Transfer</id><description>Technology Transfer</description><name>Technology Transfer</name><class>wt:fundingStream</class><parent/></funding_level_0></fundingtree>"))))

				.build();

	}

	public static ExtraInfo extraInfo(final String name, final String provenance, final String trust, final String typology, final String value) {

		final ExtraInfo.Builder e = ExtraInfo.newBuilder().setName(name).setProvenance(provenance).setTrust(trust).setTypology(typology).setValue(value);

		return e.build();

	}

	public static StringField sf(final String s) {

		return sf(s, null);

	}

	public static StringField sf(final String s, final DataInfo dataInfo) {

		final StringField.Builder sf = StringField.newBuilder().setValue(s);

		if (dataInfo != null) {

			sf.setDataInfo(dataInfo);

		}

		return sf.build();

	}

	public static DNGFDecoder embed(final GeneratedMessage msg,

			final Kind kind,

			final boolean deletedByInference,

			final boolean inferred,

			final String provenance,

			final String action) {

		final DNGF.Builder dngf = DNGF.newBuilder()

				.setKind(kind)

				.setLastupdatetimestamp(System.currentTimeMillis())

				.setDataInfo(

						DataInfo.newBuilder().setDeletedbyinference(deletedByInference).setInferred(inferred).setTrust("0.5")

								.setInferenceprovenance(provenance).setProvenanceaction(getQualifier(action, action)));

		switch (kind) {

		case entity:

			dngf.setEntity((DNGFEntity) msg);

			break;

		case relation:

			dngf.setRel((DNGFRel) msg);

			break;

		default:

			break;

		}

		return DNGFDecoder.decode(dngf.build());

	}

	public static DNGFDecoder embed(final GeneratedMessage msg, final Kind kind) {

		return embed(msg, kind, false, false, "inference_provenance", "provenance_action");

	}

}

package eu.dnetlib.data.mapreduce.util;

import java.util.List;

import eu.dnetlib.data.graph.model.DNGFDecoder;

import eu.dnetlib.data.proto.KindProtos.Kind;

import org.junit.Test;

import static org.junit.Assert.assertFalse;

import static org.junit.Assert.assertNotNull;

public class DNGFDecoderTest {

	@Test

	public void testAsXml() {

		final DNGFDecoder decoder = DNGFTest.embed(DNGFTest.getResult("50|id_1"), Kind.entity);

		assertNotNull(decoder);

		assertNotNull(decoder.asXml());

		System.out.println(decoder.asXml());

	}

	@Test

	public void testGetFieldValues() {

		final DNGFDecoder decoder = DNGFTest.embed(DNGFTest.getResult("50|id_1"), Kind.entity);

		final String path = "publication/metadata/title/value";

		final List<String> titles = decoder.decodeEntity().getFieldValues(path);

		assertNotNull(titles);

		assertFalse(titles.isEmpty());

	}

}

package eu.dnetlib.data.mapreduce.util;

import eu.dnetlib.data.graph.model.DNGFRowKeyDecoder;