D-Net

#!/bin/bash

VERSION=`xmlstarlet sel -t -m "/_:project/_:version" -v "." pom.xml`

echo "using version: $VERSION"

mvn clean install -DskipTests=true;

rm -rf ~/.m2/repository/eu/dnetlib/dnet-mapreduce-jobs-assembly;

mvn assembly:assembly -DskipTests=true && mvn install:install-file -Dfile=target/dnet-mapreduce-jobs-$VERSION-jar-with-dependencies.jar -DgroupId=eu.dnetlib -DartifactId=dnet-mapreduce-jobs-assembly -Dversion=$VERSION -Dpackaging=jar

{

		"type_source": "SVN",

		"goal": "package -U source:jar",

		"url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet50/modules/dnet-mapreduce-jobs/trunk/",

		"deploy_repository": "dnet5-snapshots",

		"version": "5",

		"mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it",

		"deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet5-snapshots",

		"name": "dnet-mapreduce-jobs"

}

package eu.dnetlib.data.mapreduce.dedup.cc;

import java.util.TreeSet;

import eu.dnetlib.data.mapreduce.hbase.dedup.cc.VertexWritable;

import org.apache.commons.lang3.StringUtils;

import org.apache.hadoop.io.Text;

import org.junit.Test;

import static org.junit.Assert.assertTrue;

/**

 * Created by claudio on 15/10/15.

 */

public class VertexWritableTest {

	@Test

	public void testSerialise() {

		VertexWritable v = new VertexWritable();

		v.setActivated(true);

		v.setVertexId(new Text("a"));

		TreeSet<Text> edges = new TreeSet<Text>();

		for(int i=0; i<5; i++) {

			edges.add(new Text("" + i));

		}

		v.setEdges(edges);

		assertTrue(v.toString() != null);

		final String json = v.toJSON();

		assertTrue(StringUtils.isNotBlank(json));

		System.out.println(json);

		final VertexWritable v1 = VertexWritable.fromJSON(json);

		final String json1 = v1.toJSON();

		assertTrue(json.equals(json1));

		System.out.println(json1);

	}

}

package eu.dnetlib.data.mapreduce.dedup;

import java.util.LinkedList;

import java.util.List;

import java.util.Map;

import java.util.PriorityQueue;

import java.util.Queue;

import java.util.UUID;

import eu.dnetlib.data.mapreduce.hbase.dedup.kv.DNGFKey;

import eu.dnetlib.data.mapreduce.hbase.dli.kv.DliKey;

import org.apache.commons.io.IOUtils;

import org.junit.Assert;

import org.junit.Before;

import org.junit.Test;

import org.springframework.core.io.ClassPathResource;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import eu.dnetlib.pace.clustering.NGramUtils;

import eu.dnetlib.pace.config.Type;

import eu.dnetlib.pace.model.Field;

import eu.dnetlib.pace.model.FieldListImpl;

import eu.dnetlib.pace.model.FieldValueImpl;

import eu.dnetlib.pace.model.MapDocument;

import eu.dnetlib.pace.model.MapDocumentComparator;

public class TitleOrderingTest {

	private List<MapDocument> results = Lists.newArrayList();

	@Before

	public void setUp() throws Exception {

		final List<String> lines = IOUtils.readLines(new ClassPathResource("eu/dnetlib/data/mapreduce/dedup/titles.txt").getInputStream());

		for (final String title : lines) {

			final Map<String, Field> fieldMap = Maps.newHashMap();

			final FieldListImpl list = new FieldListImpl();

			list.add(new FieldValueImpl(Type.String, "title", title));

			fieldMap.put("title", list);

			results.add(new MapDocument("id-" + UUID.randomUUID(), fieldMap));

		}

	}

	@Test

	public void test() {

		final Queue<MapDocument> queue = new PriorityQueue<MapDocument>(100, new MapDocumentComparator("title"));

		queue.addAll(results);

		final Queue<MapDocument> queue2 = simplifyQueue(queue);

		while (!queue2.isEmpty()) {

			final MapDocument doc = queue2.remove();

			System.out.println(doc.values("title").stringValue());

		}

	}

	private Queue<MapDocument> simplifyQueue(final Queue<MapDocument> queue) {

		final Queue<MapDocument> q = new LinkedList<MapDocument>();

		String fieldRef = "";

		final List<MapDocument> tempResults = Lists.newArrayList();

		while (!queue.isEmpty()) {

			final MapDocument result = queue.remove();

			if (!result.values("title").isEmpty()) {

				final String field = NGramUtils.cleanupForOrdering(result.values("title").stringValue());

				if (field.equals(fieldRef)) {

					tempResults.add(result);

				} else {

					if (tempResults.size() < 5) {

						q.addAll(tempResults);

					} else {

						System.out.println("Skipped field: " + fieldRef + " - size: " + tempResults.size());

					}

					tempResults.clear();

					tempResults.add(result);

					fieldRef = field;

				}

			}

		}

		if (tempResults.size() < 5) {

			q.addAll(tempResults);

		} else {

			System.out.println("Skipped field: " + fieldRef + " - size: " + tempResults.size());

		}

		return q;

	}

    @Test

    public void compareDLIKey() {

        DNGFKey k1 = DNGFKey.mergesRel("a");

        DNGFKey k2 = DNGFKey.otherRel("a");

        Assert.assertEquals(-1, k1.compareTo(k2));

        Assert.assertEquals(0, k1.compareTo(k1));

        Assert.assertEquals(1, k2.compareTo(k1));

    }

}

package eu.dnetlib.data.mapreduce.dedup;

import java.util.List;

import java.util.UUID;

import com.google.common.base.Predicate;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import eu.dnetlib.data.proto.DatasetProtos.Dataset;

import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty;

import eu.dnetlib.data.proto.KindProtos.Kind;

import eu.dnetlib.data.proto.DNGFProtos.DNGF;

import eu.dnetlib.data.proto.DNGFProtos.DNGFEntity;

import eu.dnetlib.data.proto.DNGFProtos.DNGFEntity.Builder;

import eu.dnetlib.data.proto.PublicationProtos.Publication;

import eu.dnetlib.data.proto.TypeProtos.Type;

import eu.dnetlib.data.transform.DNGFEntityMerger;

import org.junit.Before;

import org.junit.Test;

import static eu.dnetlib.data.graph.model.DNGFUtils.*;

public class DNGFMergeTest {

	private List<DNGF> oafList;

	private DNGFEntityMerger merger;

	@Before

	public void setUp() throws Exception {

		merger = new DNGFEntityMerger();

		oafList = Lists.newArrayList();

		oafList.add(getDNGF("0.1").setEntity(

				getEntity("id_1", null, "pid_1").setPublication(

						Publication.newBuilder().setMetadata(

                                Publication.Metadata.newBuilder().setDateofacceptance(sf("2012-01-01"))

                                        .addTitle(sp("vvvv Title", "main title", "dnet:dataCite_title"))

                                        .setResulttype(simpleQualifier("publication", "dnet:result_typologies"))))).build());

        oafList.add(getDNGF("0.1").setEntity(

				getEntity("id_2", "originalId_2", "pid_2").setPublication(

						Publication.newBuilder().setMetadata(

                                Publication.Metadata.newBuilder().setDateofacceptance(sf(""))

                                        .addTitle(sp("aaaa Title", "main title", "dnet:dataCite_title"))

                                        .setResulttype(simpleQualifier("publication", "dnet:result_typologies"))))).build());

        oafList.add(getDNGF("0.2").setEntity(

				getEntity("id_3", "originalId_2", "pid_2").setPublication(

						Publication.newBuilder().setMetadata(

                                Publication.Metadata.newBuilder().addTitle(sp("cccc Title", "sub title", "dnet:dataCite_title"))

                                        .setResulttype(simpleQualifier("publication", "dnet:result_typologies"))))).build());

		oafList.add(getDNGF("0.3").setEntity(

				getEntity("id_$", null, "pid_3").setPublication(

						Publication.newBuilder().setMetadata(

                                Publication.Metadata.newBuilder().setPublisher(sf("AMER CHEMICAL SOCXXXXXXXXXXXXXXXXX"))

                                        .setResulttype(simpleQualifier("publication", "dnet:result_typologies"))))).build());

        oafList.add(getDNGF("0.5").setEntity(

				getEntity("id_5", null, null).setDataset(

						Dataset.newBuilder().setMetadata(

                                Dataset.Metadata.newBuilder().addTitle(sp("hhhh title", "main title", "dnet:dataCite_title"))

                                        .setPublisher(sf("AMER CHEMICAL SOC X"))

                                        .setResulttype(simpleQualifier("publication", "dnet:result_typologies")).setStoragedate(sf("2012-11-18"))

                                        .setLanguage(simpleQualifier("eng", "dnet:languages")).addDescription(sf("original description")))))

                .build());

		oafList.add(getDNGF("0.6").setEntity(

				getEntity("id_6", null, "pid_6").setPublication(

						Publication.newBuilder().setMetadata(

                                Publication.Metadata.newBuilder().setResulttype(simpleQualifier("publication", "dnet:result_typologies"))

                                        .addDescription(sf("new description"))))).build());

    }

	@Test

	public void test_merge() {

		final DNGF.Builder builder = DNGF.newBuilder();

		for (final DNGF oaf : oafList) {

			builder.mergeFrom(oaf);

		}

		final Publication.Metadata.Builder metadata = builder.getEntityBuilder().getPublicationBuilder().getMetadataBuilder();

		final Iterable<StructuredProperty> filter = Iterables.filter(metadata.getTitleList(), new Predicate<StructuredProperty>() {

			@Override

			public boolean apply(final StructuredProperty sp) {

				return (sp.getQualifier() != null) && sp.getQualifier().getClassname().equals("main title");

			}

		});

		final StructuredProperty last = Iterables.getLast(filter);

		metadata.clearTitle().addAllTitle(Lists.newArrayList(last));

		System.out.println(builder.build().toString());

	}

	@Test

	public void test_merger() {

		final DNGF merge = merger.mergeEntities(null, "id", oafList).build();

		System.out.println(merge.toString());

	}

	// @Test

	// public void test_sort() {

	// Queue<Publication> q = new PriorityQueue<Publication>(3, DedupReducer.cmp);

	// for (DNGF oaf : oafList) {

	// q.add(oaf.getEntity().getPublication());

	// }

	//

	// while (!q.isEmpty()) {

	// Publication r = q.remove();

	// List<StructuredProperty> titles = r.getMetadata().getTitleList();

	// if (!titles.isEmpty()) {

	// System.out.println(titles.get(0).getValue());

	// }

	// }

	// }

	private DNGF.Builder getDNGF(final String trust) {

        return DNGF.newBuilder().setKind(Kind.entity).setDataInfo(di(trust)).setLastupdatetimestamp(System.currentTimeMillis());

    }

	private DNGFEntity.Builder getEntity(final String id, final String originalId, final String pid) {

		final Builder entity =

				DNGFEntity.newBuilder().setType(Type.publication).setId(id).addOriginalId(originalId != null ? originalId : UUID.randomUUID().toString());

		if (pid != null) {

            entity.addPid(sp(pid, "class", "scheme"));

        }

		return entity;

	}

}

package eu.dnetlib.data.mapreduce.hbase.index.config;

import static org.junit.Assert.assertFalse;

import static org.junit.Assert.assertNotNull;

import static org.junit.Assert.assertTrue;

import java.io.IOException;

import java.util.Map.Entry;

import org.apache.commons.io.IOUtils;

import org.dom4j.DocumentException;

import org.junit.Before;

import org.junit.Test;

public class ContextMapperTest {

	private ContextMapper contextMapper;

    public static String loadContext() throws IOException {

        return IOUtils.toString(ContextMapperTest.class.getResource("context.xml").openStream());

    }

	@Before

	public void setUp() throws Exception {

        contextMapper = ContextMapper.fromXml(loadContext());

        assertNotNull(contextMapper);

	}

	@Test

	public void test() throws DocumentException {

		assertTrue(contextMapper.size() > 0);

		for (Entry<String, ContextDef> entry : contextMapper.entrySet()) {

			ContextDef def = entry.getValue();

			try {

				assertNotNull(def.getId());

				assertNotNull(def.getLabel());

				assertNotNull(def.getName());

				assertFalse(def.getId().isEmpty());

				assertFalse(def.getLabel().isEmpty());

				assertFalse(def.getName().isEmpty());

				if (def.getName().equals("context")) {

					assertNotNull(def.getType());

					assertFalse(def.getType().isEmpty());

				}

			} catch (Throwable e) {

				System.out.println("aaa");

			}

		}

	}

}

package eu.dnetlib.data.mapreduce.hbase.index.config;

import eu.dnetlib.data.graph.utils.RelDescriptor;

import eu.dnetlib.data.proto.TypeProtos.Type;

import eu.dnetlib.data.transform.OntologyLoader;

import org.apache.commons.io.IOUtils;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.io.SAXReader;

import org.junit.Test;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import static org.junit.Assert.*;

public class IndexConfigTest {

    private final static String entityGrouperClassPath = "/eu/dnetlib/bootstrap/profiles/EntityGrouperConfigurationDSResources/EntityGrouperConfigurationDSResourceType/c2b7fa8b-a3a3-439b-9799-02ea17e4d598.xml";

    public static String loadConfiguration() {

        return loadConfiguration(entityGrouperClassPath);

    }

    private static String loadConfiguration(final String classPathResource) {

        InputStream profile = OntologyLoader.class.getResourceAsStream(classPathResource);

        SAXReader saxReader = new SAXReader();

        Document doc = null;

        try {

            doc = saxReader.read(profile);

        } catch (DocumentException e) {

            e.printStackTrace();

            throw new RuntimeException(e);

        }

        return doc.valueOf("//GROUP").trim();

    }

    public static ContextMapper loadContextFromCp() throws IOException, DocumentException {

	    final InputStream is = ContextMapper.class.getResourceAsStream("context.xml");

    	return ContextMapper.fromXml(IOUtils.toString(new InputStreamReader(is)));

    }

    @Test

    public void testParse() {

        final String config = loadConfiguration(entityGrouperClassPath);

        final IndexConfig conf = IndexConfig.load(config);

        assertNotNull(conf);

    }

    @Test

    public void testLoadEntityConfTable() {

        final String config = loadConfiguration(entityGrouperClassPath);

        final IndexConfig conf = IndexConfig.load(config);

        final EntityConfigTable map = conf.getConfigMap();

        assertNotNull(map);

        assertTrue(map.size() > 0);

        for (final Type t : Type.values()) {

            assertTrue(map.keySet().contains(t));

            assertNotNull(map.get(t).getIndexDuplicates());

            for (final LinkDescriptor ld : map.get(t).getLinks().values()) {

                assertNotNull(ld.getFields());

                assertNotNull(ld.getTarget());

                final RelDescriptor relDescriptor = ld.getRelDescriptor();

                assertNotNull(relDescriptor);

                assertNotNull(relDescriptor.getCode());

                assertFalse(relDescriptor.getCode().isEmpty());

                assertNotNull(relDescriptor.getOntologyCode());

                assertNotNull(relDescriptor.getTermCode());

                // System.out.println(ld);

            }

        }

    }

}

package eu.dnetlib.data.mapreduce.hbase.oai;

import static org.junit.Assert.assertFalse;

import static org.junit.Assert.assertNotNull;

import static org.junit.Assert.assertTrue;

import java.io.IOException;

import java.text.ParseException;

import java.util.Date;

import org.apache.commons.io.IOUtils;

import org.apache.hadoop.mapreduce.Counter;

import org.apache.solr.common.util.DateUtil;

import org.dom4j.DocumentException;

import org.junit.Before;

import org.junit.Test;

import org.mockito.Matchers;

import org.mockito.Mock;

import org.mockito.Mockito;

import org.mockito.MockitoAnnotations;

import org.springframework.core.io.ClassPathResource;

import org.springframework.core.io.Resource;

import com.google.common.collect.ArrayListMultimap;

import com.google.common.collect.Lists;

import com.google.common.collect.Multimap;

import com.mongodb.DBCollection;

import com.mongodb.DBObject;

import eu.dnetlib.data.mapreduce.hbase.oai.config.OAIConfiguration;

import eu.dnetlib.data.mapreduce.hbase.oai.config.OAIConfigurationReader;

import eu.dnetlib.data.mapreduce.hbase.oai.config.OAIConfigurationStringReader;

import eu.dnetlib.data.mapreduce.hbase.oai.utils.MongoSetCollection;

import eu.dnetlib.data.mapreduce.hbase.oai.utils.RecordFieldsExtractor;

import eu.dnetlib.miscutils.datetime.DateUtils;

public class OAIFeedMapperTest {

	private OaiFeedMapper oaiFeedMapper;

	/** File path where to find the XML configuration profile **/

	private String pathToProfile = "eu/dnetlib/data/mapreduce/hbase/oai/config/OAIPublisherConfiguration-1.xml";

	private OAIConfigurationStringReader oaiConfigurationReader;

	private OAIConfiguration oaiConfiguration;

	@Mock

	private MongoSetCollection mongoSetCollection;

	@Mock

	private DBCollection discardedCollection;

	@SuppressWarnings("rawtypes")

	@Mock

	private org.apache.hadoop.mapreduce.Mapper.Context context;

	@Mock

	private Counter counter;

	private Date feedDate;

	private String objId1 = "oai:dnet:openaire____::2fa6b215ace86e409dde3ba4b2a6b504";

	private String goodRecord = "<?xml version=\"1.0\"?>\n<record>\n  <result xmlns:dc=\"http://purl.org/dc/elements/1.1/\"\n    xmlns:dnet=\"eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions\"\n    xmlns:dr=\"http://www.driver-repository.eu/namespace/dr\"\n    xmlns:dri=\"http://www.driver-repository.eu/namespace/dri\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">\n    <header>\n      <dri:objIdentifier>openaire____::2fa6b215ace86e409dde3ba4b2a6b504</dri:objIdentifier>\n      <dri:repositoryId/>\n      <dri:dateOfCollection>2013-10-09</dri:dateOfCollection>\n    </header>\n    <metadata>\n      <oaf:entity xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" \n\t\t    xmlns:oaf=\"http://namespace.openaire.eu/oaf\" \n\t\t    xsi:schemaLocation=\"http://namespace.openaire.eu/oaf http://www.openaire.eu/schema/0.1/oaf-0.1.xsd\">\n\t\t<oaf:datasource>\n\t\t\t<officialname>The Internet Journal of Orthopedic Surgery</officialname><englishname>The Internet Journal of Orthopedic Surgery</englishname><websiteurl>http://www.ispub.com/journal/the-internet-journal-of-orthopedic-surgery/</websiteurl><accessinfopackage/><namespaceprefix>issn15312968</namespaceprefix><datasourcetypeui classid=\"pubsrepository::journal\" classname=\"pubsrepository::journal\" schemeid=\"dnet:datasource_typologies\" schemename=\"dnet:datasource_typologies\"/><datasourcetype classid=\"pubsrepository::journal\" classname=\"pubsrepository::journal\" schemeid=\"dnet:datasource_typologies\" schemename=\"dnet:datasource_typologies\"/><openairecompatibility classid=\"notCompatible\" classname=\"notCompatible\" schemeid=\"dnet:compatibilityLevel\" schemename=\"dnet:compatibilityLevel\"/><latitude>0.0</latitude><longitude>0.0</longitude><subjects/><policies name=\"\" id=\"\"/><logourl/><contactemail/><dateofvalidation/><description/><odnumberofitems/><odnumberofitemsdate/><odpolicies/><odlanguages/><odcontenttypes/><releasestartdate/><releaseenddate/><missionstatementurl/><dataprovider>false</dataprovider><serviceprovider>false</serviceprovider><databaseaccesstype/><datauploadtype/><databaseaccessrestriction/><datauploadrestriction/><versioning>false</versioning><citationguidelineurl/><qualitymanagementkind/><pidsystems/><certificates/><originalId>openaire____::issn15312968</originalId><collectedfrom name=\"DOAJ-Articles\" id=\"driver______::bee53aa31dc2cbb538c10c2b65fa5824\"/><pid/><datainfo><inferred>false</inferred><deletedbyinference>false</deletedbyinference><trust>0.9</trust><inferenceprovenance/><provenanceaction classid=\"UNKNOWN\" classname=\"UNKNOWN\" schemeid=\"dnet:provenanceActions\" schemename=\"dnet:provenanceActions\"/></datainfo>\n\t\t  <rels>\n\t\t  </rels>\n\t\t  <children>\n\t\t  </children>\n\t\t</oaf:datasource>\n      </oaf:entity>\n    </metadata>\n  </result>\n</record>";

	private String dedupedRecord = "dedupedRecord.xml";

	private String representativeRecord = "representativeRecord.xml";

	@Before

	public void setUp() throws Exception {

		MockitoAnnotations.initMocks(this);

		oaiFeedMapper = new OaiFeedMapper();

		Resource resource = new ClassPathResource(pathToProfile);

		// setting up the parser and the profile as a string

		String configurationProfile = IOUtils.toString(resource.getInputStream());

		System.out.println("oaiConfiguration:\n" + configurationProfile);

		oaiConfigurationReader = new OAIConfigurationStringReader(configurationProfile);

		oaiConfiguration = oaiConfigurationReader.getOaiConfiguration();

		String feedDateString = DateUtils.now_ISO8601();

		try {

			feedDate = DateUtil.parseDate(feedDateString);

		} catch (ParseException e) {

			e.printStackTrace(System.err);

			throw new RuntimeException(e);

		}

		oaiFeedMapper.setFeedDate(feedDate);

		oaiFeedMapper.setMongoSetCollection(mongoSetCollection);

		oaiFeedMapper.setOaiConfiguration(oaiConfiguration);

		oaiFeedMapper.setOaiConfigurationReader(oaiConfigurationReader);

		oaiFeedMapper.setDiscardedCollection(discardedCollection);

		oaiFeedMapper.setSkipDuplicates(true);

		oaiFeedMapper.setDuplicateXPath("//entity//datainfo/deletedbyinference[./text() = 'true']");

		Mockito.when(mongoSetCollection.normalizeSetSpec(Matchers.anyString())).thenReturn("aNormalisedSetName");

		Mockito.doNothing().when(counter).increment(Matchers.anyLong());

		Mockito.when(context.getCounter(Matchers.anyString(), Matchers.anyString())).thenReturn(counter);

		Mockito.when(discardedCollection.insert((DBObject) Matchers.any())).thenReturn(null);

	}

	@Test

	public void testCreateBasicObject() throws DocumentException, IOException, InterruptedException {

		RecordFieldsExtractor parser = new RecordFieldsExtractor(Lists.newArrayList(oaiConfiguration.getFieldsFor("oaf", "index", "openaire")));

		Multimap<String, String> parsedRecord = parser.extractFields(goodRecord, oaiConfiguration.getEnrichmentXPathsFor("oaf", "index", "openaire"));

		DBObject obj = oaiFeedMapper.createBasicObject(objId1, goodRecord, parsedRecord, context);

		// NOTE that LAST_COLLECTION_DATE_FIELD, DATESTAMP_FIELD,UPDATED_FIELD are not set by the method we are testing, but by the caller

		// method (handleRecord) because they values to set depend on the record status (NEW|UPDATED|UNCHANGED)

		System.out.println(obj);

		assertNotNull(obj);

	}

	@Test

	public void testCreateBasicObjectRep() throws DocumentException, IOException, InterruptedException {

		RecordFieldsExtractor parser = new RecordFieldsExtractor(Lists.newArrayList(oaiConfiguration.getFieldsFor("oaf", "index", "openaire")));

		String repRecordString = IOUtils.toString(this.getClass().getResourceAsStream(representativeRecord));

		Multimap<String, String> parsedRecord = parser.extractFields(repRecordString, oaiConfiguration.getEnrichmentXPathsFor("oaf", "index", "openaire"));

		DBObject obj = oaiFeedMapper.createBasicObject(objId1, repRecordString, parsedRecord, context);

		// NOTE that LAST_COLLECTION_DATE_FIELD, DATESTAMP_FIELD,UPDATED_FIELD are not set by the method we are testing, but by the caller

		// method (handleRecord) because they values to set depend on the record status (NEW|UPDATED|UNCHANGED)

		System.out.println(obj);

		assertNotNull(obj);

	}

	@Test

	public void testParseDeduplicated() throws IOException {

		RecordFieldsExtractor parser = new RecordFieldsExtractor(Lists.newArrayList(oaiConfiguration.getFieldsFor("oaf", "index", "openaire")));

		String dedupedRecordString = IOUtils.toString(this.getClass().getResourceAsStream(dedupedRecord));

		parser.setSkipDuplicates(true);

		parser.setDuplicateXPath("//*[local-name()='entity']//*[local-name()='datainfo']/*[local-name()='deletedbyinference'][./text() = 'true']");

		Multimap<String, String> parsedRecord = parser.extractFields(dedupedRecordString, oaiConfiguration.getEnrichmentXPathsFor("oaf", "index", "openaire"));

		assertFalse(oaiFeedMapper.checkRecordFields(parsedRecord, context, "x", dedupedRecordString));

	}

	@Test

	public void testParseDeduplicated2() throws IOException {

		RecordFieldsExtractor parser = new RecordFieldsExtractor(Lists.newArrayList(oaiConfiguration.getFieldsFor("oaf", "index", "openaire")));

		String dedupedRecordString = IOUtils.toString(this.getClass().getResourceAsStream(dedupedRecord));

		parser.setSkipDuplicates(true);

		parser.setDuplicateXPath("//x");

		Multimap<String, String> parsedRecord = parser.extractFields(dedupedRecordString, oaiConfiguration.getEnrichmentXPathsFor("oaf", "index", "openaire"));

		assertTrue(oaiFeedMapper.checkRecordFields(parsedRecord, context, "x", dedupedRecordString));

	}

	@Test

	public void testCheckRecordFieldsDeduplicated() {

		Multimap<String, String> recordProps = ArrayListMultimap.create();

		recordProps.put("duplicate", "true");

		assertFalse(oaiFeedMapper.checkRecordFields(recordProps, context, objId1, goodRecord));

	}

	@Test

	public void testCheckRecordFieldsNotDeduplicated() {

		Multimap<String, String> recordProps = ArrayListMultimap.create();

		recordProps.put("duplicate", "false");

		recordProps.put(OAIConfigurationReader.ID_FIELD, "10|xxxx");

		assertTrue(oaiFeedMapper.checkRecordFields(recordProps, context, objId1, goodRecord));

	}

	@Test

	public void testCheckRecordFieldsEmpty() {

		Multimap<String, String> recordProps = ArrayListMultimap.create();

		assertFalse(oaiFeedMapper.checkRecordFields(recordProps, context, objId1, goodRecord));

	}

	@Test

	public void testCheckRecordFieldsNull() {

		Multimap<String, String> recordProps = null;

		assertFalse(oaiFeedMapper.checkRecordFields(recordProps, context, objId1, goodRecord));

	}

}

package eu.dnetlib.data.mapreduce.hbase.oai;

import static org.junit.Assert.assertTrue;

import java.io.IOException;

import java.io.StringReader;

import java.util.Collection;

import java.util.List;

import org.apache.commons.io.IOUtils;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.io.SAXReader;

import org.junit.Before;

import org.junit.Test;

import com.google.common.collect.Lists;

import eu.dnetlib.data.mapreduce.hbase.oai.utils.RecordFieldsExtractor;

public class RecordFieldsExtractorTest {

	private String record = "representativeRecord.xml";

	private String record2 = "dedupedRecord.xml";

	private String record3 = "originalRecord.xml";

	private RecordFieldsExtractor fieldExtractor;

	private List<String> enrichmentXPaths = Lists.newArrayList("//subject[./@inferred='true']", "//result/datainfo[./inferenceprovenance='dedup']",

			"//rel[./@inferred='true' and ./@inferenceprovenance != 'dedup']");

	@Before

	public void setUp() throws Exception {

		fieldExtractor = new RecordFieldsExtractor();

	}

	@Test

	public void testEnhanced() throws IOException, DocumentException {

		String recordString = IOUtils.toString(this.getClass().getResourceAsStream(record));

		Document doc = new SAXReader().read(new StringReader(recordString));

		Collection<String> sets = fieldExtractor.getEnrichedSets(doc, enrichmentXPaths, Lists.newArrayList("set1", "set2"));

		System.out.println(sets);

		assertTrue(sets.contains("set1_enriched") && sets.contains("set2_enriched"));

	}

	@Test

	public void testEnhancedDeduped() throws IOException, DocumentException {

		String recordString = IOUtils.toString(this.getClass().getResourceAsStream(record2));

		Document doc = new SAXReader().read(new StringReader(recordString));

		Collection<String> sets = fieldExtractor.getEnrichedSets(doc, enrichmentXPaths, Lists.newArrayList("set1", "set2"));

		System.out.println(sets);

		assertTrue(sets.contains("set1_enriched") && sets.contains("set2_enriched"));

	}

	@Test

	public void testNotEnhanced() throws IOException, DocumentException {

		String recordString = IOUtils.toString(this.getClass().getResourceAsStream(record3));

		Document doc = new SAXReader().read(new StringReader(recordString));

		Collection<String> sets = fieldExtractor.getEnrichedSets(doc, enrichmentXPaths, Lists.newArrayList("set1", "set2"));

		System.out.println(sets);

		assertTrue(sets.isEmpty());

	}

}

package eu.dnetlib.data.mapreduce.hbase.oai.config;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertNotNull;

import static org.junit.Assert.assertTrue;

import java.io.IOException;

import java.util.List;

import java.util.Map;

import java.util.Map.Entry;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.junit.Before;

import org.junit.Test;

import org.springframework.core.io.ClassPathResource;

import org.springframework.core.io.Resource;

import com.google.common.collect.ArrayListMultimap;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import com.google.common.collect.Multimap;

import eu.dnetlib.data.mapreduce.hbase.oai.utils.MDFInfo;

import eu.dnetlib.data.mapreduce.hbase.oai.utils.PublisherField;

import eu.dnetlib.data.mapreduce.hbase.oai.utils.SetInfo;

public class OAIConfigurationParserTest {

	private static final Log log = LogFactory.getLog(OAIConfigurationParserTest.class); // NOPMD by marko on 11/24/08 5:02 PM

	/** Resource under test **/

	private OAIConfigurationParser parser;

	/** Configuration profile to parse **/

	private String configurationProfile;

	/** File path where to find the XML configuration profile **/

	private String pathToProfile = "eu/dnetlib/data/mapreduce/hbase/oai/config/OAIPublisherConfiguration-1.xml";

	/** Expected configuration based on the profile available at the given pathToProfile. **/

	private OAIConfiguration expectedConfiguration;

	@Before

	public void setUp() throws Exception {

		// preparing the expected configuration that we will use to perform asserts on the result of parsing

		expectedConfiguration = new OAIConfiguration();

		expectedConfiguration.setIdScheme("oai");

		expectedConfiguration.setIdNamespace("dnet");

		Map<String, SetInfo> setsMap = Maps.newHashMap();

		setsMap.put("set1", new SetInfo("set1", "set number 1", "test set", "(set =\"aSet\")", true));

		expectedConfiguration.setSetsMap(setsMap);

		expectedConfiguration.setFieldNames(Lists.newArrayList("title", "objIdentifier", "set"));

		Map<String, MDFInfo> formatsMap = Maps.newHashMap();

		formatsMap.put("oai_dc", new MDFInfo("oai_dc", "SCHEMA", "NAMESPACE", "oai_dc", "store", "driver", "", true));

		formatsMap.put("hope", new MDFInfo("hope", "SCHEMA", "NAMESPACE", "hope", "L", "I", "country = IT", "tdsRule", true));

		expectedConfiguration.setMdFormatsMap(formatsMap);

		List<PublisherField> fields = Lists.newArrayList();

		String oaidcKey = "oai_dc-store-driver";

		String hopeKey = "hope-L-I";

		PublisherField titleField = new PublisherField();

		titleField.setFieldName("title");

		titleField.setRepeatable(true);

		Multimap<String, String> titleSources = ArrayListMultimap.create();

		titleSources.put(oaidcKey, "//*[local-name() = 'title']");

		titleSources.put(oaidcKey, "//*[local-name() ='alternative']");

		titleSources.put(hopeKey, "//title");

		titleSources.put(hopeKey, "//titleinfo");

		titleField.setSources(titleSources);

		fields.add(titleField);

		PublisherField objIdField = new PublisherField();

		objIdField.setFieldName("objIdentifier");

		objIdField.setRepeatable(false);

		Multimap<String, String> objIdSources = ArrayListMultimap.create();

		objIdSources.put(oaidcKey, "//*[local-name() ='objIdentifier']");

		objIdField.setSources(objIdSources);

		fields.add(objIdField);

		PublisherField setField = new PublisherField();

		setField.setFieldName("set");

		setField.setRepeatable(true);

		Multimap<String, String> setSources = ArrayListMultimap.create();

		setSources.put(oaidcKey, "//*[local-name() ='repositoryId']");

		setField.setSources(setSources);

		fields.add(setField);

		expectedConfiguration.setFields(fields);

		Resource resource = new ClassPathResource(pathToProfile);

		// setting up the parser and the profile as a string

		parser = new OAIConfigurationParser();

		configurationProfile = IOUtils.toString(resource.getInputStream());

		log.info("setUp completed");

	}

	@Test

	public void testGetConfiguration() throws IOException {

		OAIConfiguration result = parser.getConfiguration(configurationProfile);

		assertNotNull(result);

		// asserts on sets

		assertTrue(result.getSetsMap().containsKey("set1"));

		SetInfo expected = expectedConfiguration.getSetsMap().get("set1");

		SetInfo parsedSet = result.getSetsMap().get("set1");

		assertEquals(expected.getQuery(), parsedSet.getQuery());

		assertEquals(expected.getSetDescription(), parsedSet.getSetDescription());

		assertEquals(expected.getSetName(), parsedSet.getSetName());

		assertEquals(expected.getSetSpec(), parsedSet.getSetSpec());

		// asserts on indices

		assertTrue(result.getFieldNames().containsAll(expectedConfiguration.getFieldNames()));

		assertEquals(3, result.getFieldsFor("oai_dc", "store", "driver").size());

		assertEquals(1, result.getFieldsFor("hope", "L", "I").size());

		// asserts on metadata formats

		assertTrue(result.getMdFormatsMap().containsKey("hope"));

		assertTrue(result.getMdFormatsMap().containsKey("oai_dc"));

		MDFInfo expectedFormat = expectedConfiguration.getMdFormatsMap().get("hope");

		MDFInfo parsedFormat = result.getMdFormatsMap().get("hope");

		assertEquals(expectedFormat.getBaseQuery(), parsedFormat.getBaseQuery());

		assertEquals(expectedFormat.getNamespace(), parsedFormat.getNamespace());

		assertEquals(expectedFormat.getPrefix(), parsedFormat.getPrefix());

		assertEquals(expectedFormat.getSchema(), parsedFormat.getSchema());

		assertEquals(expectedFormat.getSourceFormatInterpretation(), parsedFormat.getSourceFormatInterpretation());

		assertEquals(expectedFormat.getSourceFormatLayout(), parsedFormat.getSourceFormatLayout());

		assertEquals(expectedFormat.getSourceFormatName(), parsedFormat.getSourceFormatName());

		assertEquals(expectedFormat.getTransformationRuleID(), parsedFormat.getTransformationRuleID());

		assertEquals(expectedConfiguration.getIdScheme(), result.getIdScheme());

		assertEquals(expectedConfiguration.getIdNamespace(), result.getIdNamespace());

		for (PublisherField pf : result.getFields()) {

			for (Entry<String, String> e : pf.getSources().entries()) {

				assertTrue(!StringUtils.isBlank(e.getValue()));

				System.out.println(e.getValue());

			}

		}

	}

	@Test

	public void testPrintConfiguration() throws IOException {

		OAIConfiguration result = parser.getConfiguration(configurationProfile);

		System.out.println(result);

		System.out.println("\n\n");

		for (PublisherField pf : result.getFields()) {

			System.out.println(pf.toString());

		}

	}

}

package eu.dnetlib.data.mapreduce.util;

import java.util.List;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.google.protobuf.InvalidProtocolBufferException;

import com.googlecode.protobuf.format.JsonFormat;

import eu.dnetlib.data.graph.model.DNGFDecoder;

import eu.dnetlib.data.graph.model.DNGFRowKeyDecoder;

import eu.dnetlib.data.mapreduce.hbase.index.config.*;

import eu.dnetlib.data.proto.*;

import eu.dnetlib.data.proto.KindProtos.Kind;

import eu.dnetlib.data.proto.TypeProtos.Type;

import eu.dnetlib.data.transform.OntologyLoader;

import eu.dnetlib.data.transform.xml.AbstractDNetXsltFunctions;

import eu.dnetlib.miscutils.functional.xml.XMLIndenter;

import org.apache.commons.codec.binary.Base64;

import org.junit.Before;

import org.junit.Test;

import static org.junit.Assert.assertTrue;

import static eu.dnetlib.data.graph.model.DNGFUtils.*;

public class XmlRecordFactoryTest extends AbstractRecordFactoryTest {

	public static final String CITATION_XML =

			"<citations>\n  <citation>\n    <rawText>[10] M. Foret et al., Phys. Rev. B 66, 024204 (2002).</rawText>\n  </citation>\n  <citation>\n    <rawText>[11] B. Ru\175404\264e et al., Phys. Rev. Lett. 90, 095502 (2003).</rawText>\n  </citation>\n  <citation>\n    <rawText>[12] U. Buchenau et al., Phys. Rev. B 34, 5665 (1986).</rawText>\n  </citation>\n  <citation>\n    <rawText>[13] S.N. Taraskin and S.R. Elliott, J. Phys.: Condens. Mat- ter 11, A219 (1999).</rawText>\n  </citation>\n  <citation>\n    <rawText>[14] B. Hehlen et al., Phys. Rev. Lett. 84, 5355 (2000).</rawText>\n  </citation>\n  <citation>\n    <rawText>[15] N.V. Surotsev et al., J. Phys.: Condens. Matter 10, L113 (1998).</rawText>\n  </citation>\n  <citation>\n    <rawText>[16] D.A. Parshin and C. Laermans, Phys. Rev. B 63, 132203 (2001).</rawText>\n  </citation>\n  <citation>\n    <rawText>[17] V.L. Gurevich et al., Phys. Rev. B 67, 094203 (2003).</rawText>\n  </citation>\n  <citation>\n    <rawText>[18] A. Matic et al., Phys. Rev. Lett. 86, 3803 (2001).</rawText>\n  </citation>\n  <citation>\n    <rawText>[19] E. Rat et al., arXiv:cond-mat/0505558, 23 May 2005.</rawText>\n  </citation>\n  <citation>\n    <rawText>[1] R.C. Zeller and R.O. Pohl, Phys. Rev. B 4, 2029 (1971).</rawText>\n  </citation>\n  <citation>\n    <rawText>[20] C.A. Angell, J. Non-Cryst. Solids 131\20023133, 13 (1991).</rawText>\n  </citation>\n  <citation>\n    <rawText>[21] A.P. Sokolov et al., Phys. Rev. Lett. 71, 2062 (1993).</rawText>\n  </citation>\n  <citation>\n    <rawText>[22] T. Matsuo et al., Solid State Ionics 154-155, 759 (2002).</rawText>\n  </citation>\n  <citation>\n    <rawText>[23] V.K. Malinovsky et al., Europhys. Lett. 11, 43 (1990).</rawText>\n  </citation>\n  <citation>\n    <rawText>[24] J. Lor\250osch et al., J. Non-Cryst. Solids 69, 1 (1984).</rawText>\n  </citation>\n  <citation>\n    <rawText>[25] U. Buchenau, Z. Phys. B 58, 181 (1985).</rawText>\n  </citation>\n  <citation>\n    <rawText>[26] A.F. Io\175400e and A.R. Regel, Prog. Semicond. 4, 237 (1960).</rawText>\n  </citation>\n  <citation>\n    <rawText>[27] R. Dell\20031Anna et al., Phys. Rev. Lett. 80, 1236 (1998).</rawText>\n  </citation>\n  <citation>\n    <rawText>[28] D. Fioretto et al., Phys. Rev. E 59, 4470 (1999).</rawText>\n  </citation>\n  <citation>\n    <rawText>[29] U. Buchenau et al., Phys. Rev. Lett. 77, 4035 (1996).</rawText>\n  </citation>\n  <citation>\n    <rawText>[2] M. Rothenfusser et al., Phys. Rev. B 27, 5196 (1983).</rawText>\n  </citation>\n  <citation>\n    <rawText>[30] J. Mattsson et al., J. Phys.: Condens. Matter 15, S1259 (2003).</rawText>\n  </citation>\n  <citation>\n    <rawText>[31] T. Scopigno et al., Phys. Rev. Lett. 92, 025503 (2004).</rawText>\n  </citation>\n  <citation>\n    <rawText>[32] M. Foret et al., Phys. Rev. Lett. 81, 2100 (1998).</rawText>\n  </citation>\n  <citation>\n    <rawText>[33] F. Sette et al., Science 280, 1550 (1998).</rawText>\n  </citation>\n  <citation>\n    <rawText>[34] J. Wuttke et al., Phys. Rev. E 52, 4026 (1995).</rawText>\n  </citation>\n  <citation>\n    <rawText>[35] M.A. Ramos et al., Phys. Rev. Lett. 78, 82 (1997).</rawText>\n  </citation>\n  <citation>\n    <rawText>[36] G. Monaco et al., Phys. Rev. Lett. 80, 2161 (1998).</rawText>\n  </citation>\n  <citation>\n    <rawText>[37] A. T\250olle, Rep. Prog. Phys. 64, 1473 (2001).</rawText>\n  </citation>\n  <citation>\n    <rawText>[38] As the straight lines do not cross the origin, this does not 2 imply \1623 \21035 \1651 .</rawText>\n  </citation>\n  <citation>\n    <rawText>[39] A. Matic et al., Europhys. Lett. 54, 77 (2001).</rawText>\n  </citation>\n  <citation>\n    <rawText>[3] S. Hunklinger and W. Arnold, in Physical Acoustics, Vol. XII, W.P. Mason and R.N. Thurston Eds. (Academic Press, N.Y. 1976), p. 155.</rawText>\n  </citation>\n  <citation>\n    <rawText>[40] IXS data are usually not available below \1651co, mostly for experimental reasons. E.g., that the rapid onset was not evidenced in vitreous silica [27], is not indicative of its absence but rather of a low qco \21074 1 nm\210221.</rawText>\n  </citation>\n  <citation>\n    <rawText>[41] G. Ruocco et al., Phys. Rev. Lett. 83, 5583 (1999).</rawText>\n  </citation>\n  <citation>\n    <rawText>[42] D. C\1307 iplys et al., J. Physique (Paris) 42, C6-184 (1981).</rawText>\n  </citation>\n  <citation>\n    <rawText>[43] R. Vacher et al., Rev. Sci. Instrum. 51, 288 (1980).</rawText>\n  </citation>\n  <citation>\n    <rawText>[44] R. Vacher et al., arXiv:cond-mat/0505560, 23 May 2005.</rawText>\n  </citation>\n  <citation>\n    <rawText>[45] T.N. Claytor et al., Phys. Rev. B 18, 5842 (1978).</rawText>\n  </citation>\n  <citation>\n    <rawText>[46] M. Arai et al., Physica B 263-264, 268 (1999).</rawText>\n  </citation>\n  <citation>\n    <rawText>[4] R. Vacher et al., J. Non-Cryst. Solids 45, 397 (1981); T.C. Zhu et al., Phys. Rev. B 44, 4281 (1991).</rawText>\n  </citation>\n  <citation>\n    <rawText>[5] J.E. Graebner et al., Phys. Rev. B 34, 5696 (1986).</rawText>\n  </citation>\n  <citation>\n    <rawText>[6] E. Duval and A. Mermet, Phys. Rev. B 58, 8159 (1998).</rawText>\n  </citation>\n  <citation>\n    <rawText>[7] A. Matic et al., Phys. Rev. Lett. 93, 145502 (2004).</rawText>\n  </citation>\n  <citation>\n    <rawText>[8] Often alluded to, e.g. in the Encyclopedia of Materials: Science and Technology, K.H.J. Buschow et al., Eds., Vol. 1 (Elsevier, Oxford, 2001), articles by S.R. Elliott on pp. 171-174 and U. Buchenau on pp. 212-215.</rawText>\n  </citation>\n  <citation>\n    <rawText>[9] E. Rat et al., Phys. Rev. Lett. 83, 1355 (1999).</rawText>\n  </citation>\n</citations>";

	public static final String STATISTICS_JSON =

			"[{ \"citationsPerYear\": \"many\", \"anotherCoolStatistic\": \"WoW\", \"nestedStat\": { \"firstNestedStat\" : \"value 1\", \"secondNestedStat\" : \"value 2\"}, \"listingStat\" : [ \"one\", \"two\" ] }]";

	public static final String SCHEMA_LOCATION = "http://namespace.openaire.eu/DNGF http://www.openaire.eu/schema/0.2/DNGF-0.2.xsd";

	private XmlRecordFactory builder;

	@Before

	public void setUp() throws Exception {

		builder =

				new XmlRecordFactory(IndexConfig.load(IndexConfigTest.loadConfiguration()).getConfigMap(),

						ContextMapper.fromXml(ContextMapperTest.loadContext()), OntologyLoader.loadOntologiesFromCp(),

						SCHEMA_LOCATION, true, false, false);

	}

	@Test

	public void testJsonProtobuf() {

		final DNGFDecoder decoder = embed(getPublication("id"), Kind.entity);

		final String json = JsonFormat.printToString(decoder.getDNGF());

		System.out.println(json);

		System.out.println("json size: " + json.length());

		System.out.println("binary size: " + decoder.getDNGF().toByteArray().length);

		final String base64String = Base64.encodeBase64String(decoder.getDNGF().toByteArray());

		System.out.println("base64 size: " + base64String.length());

		System.out.println("decoded " + JsonFormat.printToString(DNGFDecoder.decode(Base64.decodeBase64(base64String)).getDNGF()));

	}

	@Test

	public void testProjectFP7() throws InvalidProtocolBufferException {

		final String projectId = "40|ec__________::20012100000000000000000000000000";

		final String orgId =     "20|WOS_________::organizationId000000000000000000";

		builder.setMainEntity(embed(getProjectFP7(projectId, "SP3"), Kind.entity, false, false, "", "corda"));

		builder.addRelation(Type.organization, embed(getProjectOrganization(orgId, projectId, "isParticipant"), Kind.relation));

		DNGFProtos.DNGFRel projectPerson = getProjectPerson("30|WOS_________::personId000000000000000000000000", projectId, "isContact");

		builder.addRelation(Type.person, embed(projectPerson, Kind.relation));

		final String xml = builder.build();

		System.out.println(XMLIndenter.indent(xml));

	}

	@Test

	public void testOrganization() throws InvalidProtocolBufferException {

		final String projectId = "40|ec__________::20012100000000000000000000000000";

		final String orgIdRoot = "20|org_________::organizationIdRoot00000000000000";

		final String orgIdDup =  "20|org_________::organizationIdDup000000000000000";

		builder.setMainEntity(embed(getOrganization(orgIdRoot), Kind.entity, false, false, "", "corda"));

		builder.addRelation(Type.project, embed(getProjectOrganization(projectId, orgIdRoot, "hasParticipant"), Kind.relation));

		builder.addRelation(Type.datasource, embed(getDatasourceOrganization("10|dts_________::datasourceId00000000000000000000", orgIdRoot, "provides"), Kind.relation));

		builder.addChild(Type.organization, embed(getOrganizationOrganization(orgIdDup, orgIdRoot, "isMergedIn"), Kind.relation));

		final String xml = builder.build();

		System.out.println(XMLIndenter.indent(xml));

	}

	@Test

	public void testResultFP7() throws InvalidProtocolBufferException {

		final String resultId =   "50|WOS_________::00010000000000000000000000000000";

		final String projectId1 = "40|ec__________::20012100000000000000000000000000";

		final String projectId2 = "40|ec__________::20012200000000000000000000000000";

		builder.setMainEntity(embed(getPublication(resultId), Kind.entity, false, false, "", "pubmed"));

		builder.addRelation(Type.person, embed(getPublicationPerson("30|WOS_________::0001name000000000000000000000000", resultId, "isAuthorOf"), Kind.relation));

		builder.addRelation(Type.project,

				embed(getPublicationProject(projectId1, resultId, getProjectFP7(projectId1, "SP3"), "produces"), Kind.relation));

		builder.addRelation(Type.project,

				embed(getPublicationProject(projectId2, resultId, getProjectFP7(projectId2, "SP2"), "produces"), Kind.relation));

		builder.addRelation(Type.publication,

				embed(getSimilarityRel("50|WOS_________::00020000000000000000000000000000", resultId, getPublication(resultId), "isAmongTopNSimilarDocuments"),

						Kind.relation));

		builder.addChild(Type.publication, embed(getDedupRel("50|WOS_________::anotherResultId00000000000000000", resultId, "publication_publication", "isMergedIn"), Kind.relation));

		// System.err.println(builder.toString());

		System.out.println(XMLIndenter.indent(builder.build()));

		// System.out.println(builder.build());

	}

	@Test

	public void testResultMerged() throws InvalidProtocolBufferException {

		final String resultId = "50|WOS_________::00010000000000000000000000000000";

		final String similarResultId = "50|WOS_________::00020000000000000000000000000000";

		final String projectId1 = "40|EC__________::20012100000000000000000000000000";

		final String projectId2 = "40|EC__________::99999900000000000000000000000000";

		builder.setMainEntity(embed(getPublication(resultId), Kind.entity, false, true, "dedup", "pubmed"));

		builder.addRelation(Type.person, embed(getPublicationPerson("50|WOS_________::00010name00000000000000000000000", resultId, "isAuthorOf"), Kind.relation));

		builder.addRelation(Type.person, embed(getPublicationPerson("50|WOS_________::00020name00000000000000000000000", resultId, "isAuthorOf"), Kind.relation));

		builder.addRelation(Type.project,

				embed(getPublicationProject(projectId1, resultId, getProjectFP7(projectId1, "SP3"), "produces"), Kind.relation));

		builder.addRelation(Type.project,

				embed(getPublicationProject(projectId2, resultId, getProjectFP7(projectId2, "SP3"), "produces"), Kind.relation));

		builder.addRelation(Type.publication, embed(

				getSimilarityRel(similarResultId, resultId, getPublication(similarResultId), "isAmongTopNSimilarDocuments"), Kind.relation));

		builder.addChild(Type.publication, embed(getDedupRel("50|WOS_________::anotherResultId00000000000000000", resultId, "publication_publication", "merges"), Kind.relation));

		final String xml = builder.build();

		System.out.println(XMLIndenter.indent(xml));

	}

	@Test

	public void testDatasource() throws InvalidProtocolBufferException {

		final String datasourceId = "10|WOS_________::datasourceId00000000000000000000";

		final String orgId =        "20|WOS_________::organizationId000000000000000000";

		builder.setMainEntity(embed(getDatasource(datasourceId), Kind.entity, false, false, "", "opendoar"));

		builder.addRelation(Type.organization, embed(getDatasourceOrganization(orgId, datasourceId, "isProvidedBy"), Kind.relation));

		final String xml = builder.build();

		System.out.println(XMLIndenter.indent(xml));

	}

	@Test

	public void testProjectWT() throws InvalidProtocolBufferException {

		final String projectId = "40|ec__________::20012100000000000000000000000000";

		builder.setMainEntity(embed(getProjectWT(), Kind.entity, false, false, "", "wellcometrust"));

		builder.addChild(Type.organization, embed(getProjectOrganization(projectId, "20|ec__________::organizationId000000000000000000", "isParticipant"), Kind.relation));

		final String xml = builder.build();

		System.out.println(XMLIndenter.indent(xml));

	}

	@Test

	public void testResultWT() throws InvalidProtocolBufferException {

		final String resultId = "50|WOS_________::00001000000000000000000000000000";

		builder.setMainEntity(embed(getPublication(resultId), Kind.entity, false, false, "", "arxiv"));

		builder.addRelation(Type.person, embed(getPublicationPerson(  "50|WOS_________::00001name00000000000000000000000", resultId, "isAuthorOf"), Kind.relation));

		builder.addRelation(Type.project, embed(getPublicationProject("40|wt__________::08753600000000000000000000000000", resultId, getProjectWT(), "produces"), Kind.relation));

		builder.addChild(Type.publication, embed(getDedupRel(        "50|wt__________::anotherResultId00000000000000000", resultId, "publication_publication", "merges"), Kind.relation));

		final String xml = builder.build();

		System.out.println(XMLIndenter.indent(xml));

	}

	@Test

	public void testUrlFilter() throws InvalidProtocolBufferException {

		final List<String> filtered =

				Lists.newArrayList(Iterables.filter(Lists.newArrayList("http://www.google.com", "www.google.com"), AbstractDNetXsltFunctions.urlFilter));

		assertTrue(filtered.size() == 1);

	}

}

package eu.dnetlib.data.mapreduce.util;

import com.google.protobuf.Descriptors;

import com.google.protobuf.InvalidProtocolBufferException;

import eu.dnetlib.data.graph.model.DNGFRowKeyDecoder;

import eu.dnetlib.data.proto.*;

import static eu.dnetlib.data.graph.model.DNGFUtils.*;

import static eu.dnetlib.data.graph.model.DNGFUtils.sf;

import static eu.dnetlib.data.graph.model.DNGFUtils.simpleQualifier;

/**

 * Created by sandro on 1/10/17.

 */

public class AbstractRecordFactoryTest {

    public static DNGFProtos.DNGFEntity getPublication(final String id) {

        return getPublicationBuilder(id).build();

    }

    public static DNGFProtos.DNGFEntity.Builder getPublicationBuilder(final String id) {

        return DNGFProtos.DNGFEntity

                .newBuilder()

                .setType(TypeProtos.Type.publication)

                .setId(id)

                .setPublication(

                        PublicationProtos.Publication.newBuilder()

                                .setMetadata(

                                        PublicationProtos.Publication.Metadata

                                                .newBuilder()

                                                .addTitle(

                                                        sp("Analysis of cell viability in intervertebral disc: Effect of endplate permeability on cell population",

                                                                "main title", "dnet:result_titles"))

                                                .addTitle(sp("Another title", "alternative title", "dnet:result_titles"))

                                                .addSubject(sp("Biophysics", "subject", "dnet:result_sujects"))

                                                .setDateofacceptance(sf("2010-01-01")).addSource(sf("sourceA")).addSource(sf("sourceB"))

                                                .addContext(context("egi::virtual::970"))

                                                .addContext(context("egi::classification::natsc::math::applied"))

                                                .addContext(context("egi::classification::natsc::math"))

                                                .addContext(context("egi::classification::natsc"))

                                                .addContext(context("egi::classification")).addContext(context("egi"))

                                                .addDescription(sf("Responsible for making and maintaining the extracellular matrix ..."))

                                                .addDescription(sf("Another description ...")).setPublisher(sf("ELSEVIER SCI LTD"))

                                                .setResulttype(simpleQualifier("publication", "dnet:result_types"))

                                                .setLanguage(simpleQualifier("eng", "dnet:languages")))

                                .addInstance(instance("10|opendoar____::10", "Uk pubmed"))

                                .addInstance(instance("10|opendoar____::10", "arxiv")))

                .addCollectedfrom(kv("opendoar____::1064", "Oxford University Research Archive"))

                .addPid(sp("doi:74293", "doi", "dnet:pids")).addPid(sp("oai:74295", "oai", "dnet:pids"))

                .setDateofcollection("");

    }

    public static FieldTypeProtos.Instance.Builder instance(final String hostedbyId, final String hostedbyName) {

        return FieldTypeProtos.Instance.newBuilder()

                .setHostedby(kv(hostedbyId, hostedbyName))

                .setLicence(simpleQualifier("OpenAccess", "dnet:access_modes"))

                .setInstancetype(simpleQualifier("publication", "dnet:result_typologies"))

                .addUrl("webresource url");

    }

    public static DNGFProtos.DNGFEntity getProjectFP7(final String projectId, final String fundingProgram) throws InvalidProtocolBufferException {

        return DNGFProtos.DNGFEntity

                .newBuilder()

                .setType(TypeProtos.Type.project)

                .setId(projectId)

                .addCollectedfrom(kv("opendoar_1234", "UK pubmed"))

                .setProject(

                        ProjectProtos.Project.newBuilder()

                                .setMetadata(

                                        ProjectProtos.Project.Metadata

                                                .newBuilder()

                                                .setAcronym(sf("5CYRQOL"))

                                                .setTitle(sf("Cypriot Researchers Contribute to our Quality of Life"))

                                                .setStartdate(sf("2007-05-01"))

                                                .setEnddate(sf("2007-10-31"))

                                                .setEcsc39(sf("false"))

                                                .setContracttype(simpleQualifier("CSA", "ec:FP7contractTypes"))

                                                .addFundingtree(

                                                        sf("<fundingtree><funder><id>ec__________::EC</id><shortname>EC</shortname><name>European Commission</name></funder><funding_level_2><id>ec__________::EC::FP7::"

                                                                + fundingProgram

                                                                + "::PEOPLE</id><description>Marie-Curie Actions</description><name>PEOPLE</name><class>ec:program</class><parent><funding_level_1><id>ec__________::EC::FP7::"

                                                                + fundingProgram

                                                                + "</id><description>"

                                                                + fundingProgram

                                                                + "-People</description><name>"

                                                                + fundingProgram

                                                                + "</name><class>ec:specificprogram</class><parent><funding_level_0><id>ec__________::EC::FP7</id><description>SEVENTH FRAMEWORK PROGRAMME</description><name>FP7</name><parent/><class>ec:frameworkprogram</class></funding_level_0></parent></funding_level_1></parent></funding_level_2></fundingtree>"))))

                .build();

    }

    public static DNGFProtos.DNGFRel getProjectOrganization(final String source, final String target, final String relClass) throws InvalidProtocolBufferException {

        TypeProtos.Type sourceType = DNGFRowKeyDecoder.decode(source).getType();

        final DNGFProtos.DNGFRel.Builder dngfRel = DNGFProtos.DNGFRel.newBuilder()

                .setSource(source)

                .setSourceType(sourceType)

                .setTarget(target)

                .setTargetType(DNGFRowKeyDecoder.decode(target).getType())

                .setRelType(simpleQualifier(relClass, "project_organization"))

                .setChild(false);

        switch (sourceType) {

            case project:

                dngfRel.setCachedTarget(getProjectFP7(target, "SP3"));

                break;

            case organization:

                dngfRel.setCachedTarget(getOrganization(target));

                break;

            default:

                throw new IllegalStateException("dafuq");

        }

        return dngfRel.build();

    }

    public static DNGFProtos.DNGFEntity getOrganization(final String orgId) {

        return DNGFProtos.DNGFEntity

                .newBuilder()

                .setType(TypeProtos.Type.organization)

                .setId(orgId)

                .addCollectedfrom(kv("opendoar_1234", "UK pubmed"))

                .setOrganization(

                        OrganizationProtos.Organization.newBuilder().setMetadata(

                                OrganizationProtos.Organization.Metadata.newBuilder()

                                        .setLegalname(sf("CENTRE D'APPUI A LA RECHERCHE ET A LA FORMATION GIE"))

                                        .setLegalshortname(sf("CAREF")).setWebsiteurl(sf("www.caref-mali.org"))

                                        .setCountry(simpleQualifier("ML", "dnet:countries")))).build();

    }

    public static DNGFProtos.DNGFRel getProjectPerson(final String sourceId, final String targetId, final String relClass) {

        return DNGFProtos.DNGFRel

                .newBuilder()

                .setSource(sourceId)

                .setTarget(targetId)

                .setSourceType(DNGFRowKeyDecoder.decode(sourceId).getType())

                .setTargetType(DNGFRowKeyDecoder.decode(targetId).getType())

                .setRelType(simpleQualifier(relClass, "person_project"))

                .setChild(true)

                .setCachedTarget(getPerson()).build();

    }

    public static DNGFProtos.DNGFEntity getPerson() {

        return DNGFProtos.DNGFEntity

                .newBuilder()

                .setType(TypeProtos.Type.person)

                .setId("WOS:000277866500014_A._Shirazi-Adl")

                .setPerson(

                        PersonProtos.Person.newBuilder().setMetadata(

                                PersonProtos.Person.Metadata.newBuilder().addSecondnames(sf("Shirazi-Adl")).setFullname(sf("A. Shirazi-Adl"))

                                        .setEmail(sf("name.surname@gmail.com")).setPhone(sf("12345")).setNationality(simpleQualifier("EN", "dnet:countries"))))

                .build();

    }

    public static DNGFProtos.DNGFRel getDatasourceOrganization(final String source, final String target, final String relClass) throws InvalidProtocolBufferException {

        final DNGFProtos.DNGFRel.Builder oafRel = DNGFProtos.DNGFRel

                .newBuilder()

                .setSource(source)

                .setTarget(target)

                .setSourceType(DNGFRowKeyDecoder.decode(source).getType())

                .setTargetType(DNGFRowKeyDecoder.decode(target).getType())

                .setRelType(simpleQualifier(relClass, "datasource_organization"))

                .setChild(false);

        switch (DNGFRowKeyDecoder.decode(source).getType()) {

            case datasource:

                oafRel.setCachedTarget(getOrganization(target));

                break;

            case organization:

                oafRel.setCachedTarget(getDatasource(target));

                break;

            default:

                break;

        }

        return oafRel.build();

    }

    public static DNGFProtos.DNGFEntity getDatasource(final String datasourceId) {

        return DNGFProtos.DNGFEntity

                .newBuilder()

                .setType(TypeProtos.Type.datasource)

                .setId(datasourceId)

                .setDatasource(

                        DatasourceProtos.Datasource.newBuilder().setMetadata(

                                DatasourceProtos.Datasource.Metadata.newBuilder().setOfficialname(sf("officialname")).setEnglishname(sf("englishname"))

                                        .setWebsiteurl(sf("websiteurl")).setContactemail(sf("contactemail")).addAccessinfopackage(sf("accessinforpackage"))

                                        .setNamespaceprefix(sf("namespaceprofix")).setDescription(sf("description")).setOdnumberofitems(sf("numberofitems"))

                                        .setOdnumberofitemsdate(sf("numberofitems date"))

                                        // .addOdsubjects("subjects")

                                        .setOdpolicies(sf("policies")).addOdlanguages(sf("languages")).addOdcontenttypes(sf("contenttypes"))

                                        .setDatasourcetype(simpleQualifier("type class", "type scheme")))).build();

    }

    //////////////

    public static DNGFProtos.DNGFRel getOrganizationOrganization(final String source, final String target, final String relClass) {

        final DNGFProtos.DNGFRel.Builder oafRel = DNGFProtos.DNGFRel

                .newBuilder()

                .setSource(source)

                .setTarget(target)

                .setSourceType(DNGFRowKeyDecoder.decode(source).getType())

                .setTargetType(DNGFRowKeyDecoder.decode(target).getType())

                .setRelType(simpleQualifier(relClass, "organization_organization"))

                .setChild(true);

        switch (relClass) {

            case "isMergedIn":

                oafRel.setCachedTarget(getOrganization(source));

                break;

            case "merges":

                oafRel.setCachedTarget(getOrganization(target));

                break;

            default:

                break;

        }

        return oafRel.build();

    }

    public static DNGFProtos.DNGFRel getPublicationProject(final String from, final String to, final DNGFProtos.DNGFEntity project, final String relClass)

            throws InvalidProtocolBufferException {

        return DNGFProtos.DNGFRel

                .newBuilder()

                .setSource(from)

                .setTarget(to)

                .setSourceType(DNGFRowKeyDecoder.decode(from).getType())

                .setTargetType(DNGFRowKeyDecoder.decode(to).getType())

                .setRelType(simpleQualifier(relClass, "publication_project"))

                .setChild(false)

                .setCachedTarget(project).build();

    }

    public static DNGFProtos.DNGFRel getPublicationPerson(final String from, final String to, final String relClass)

            throws InvalidProtocolBufferException {

        return DNGFProtos.DNGFRel

                .newBuilder()

                .setSource(from)

                .setTarget(to)

                .setSourceType(DNGFRowKeyDecoder.decode(from).getType())

                .setTargetType(DNGFRowKeyDecoder.decode(to).getType())

                .setRelType(simpleQualifier(relClass, "person_publication"))

                .setChild(false).build();

    }

    public static DNGFProtos.DNGFRel getSimilarityRel(final String sourceId, final String targetId, final DNGFProtos.DNGFEntity result, final String relClass) {

        return DNGFProtos.DNGFRel

                .newBuilder()

                .setSource(sourceId)

                .setTarget(targetId)

                .setSourceType(DNGFRowKeyDecoder.decode(sourceId).getType())

                .setTargetType(DNGFRowKeyDecoder.decode(targetId).getType())

                .setRelType(simpleQualifier(relClass, "publication_publication"))

                .setChild(true)

                .setCachedTarget(result)

                .build();

    }

    public static DNGFProtos.DNGFRel getDedupRel(final String source, final String target, final String relType, final String relClass) {

        return DNGFProtos.DNGFRel.newBuilder()

                .setSource(source)

                .setTarget(target)

                .setSourceType(DNGFRowKeyDecoder.decode(source).getType())

                .setTargetType(DNGFRowKeyDecoder.decode(target).getType())

                .setRelType(simpleQualifier(relClass, relType))

                .setChild(true)

                .setCachedTarget(getPublication(target)).build();

    }

    public static DNGFProtos.DNGFEntity getProjectWT() throws InvalidProtocolBufferException {

        return DNGFProtos.DNGFEntity

                .newBuilder()

                .setType(TypeProtos.Type.project)

                .setId("project|wt::087536")

                .addCollectedfrom(kv("wellcomeTrust", "wellcome trust"))

                .setProject(

                        ProjectProtos.Project.newBuilder()

                                .setMetadata(

                                        ProjectProtos.Project.Metadata

                                                .newBuilder()

                                                .setAcronym(sf("UNKNOWN"))

                                                .setTitle(sf("Research Institute for Infectious Diseases of Poverty (IIDP)."))

                                                .setStartdate(sf("2007-05-01"))

                                                .setEnddate(sf("2007-10-31"))

                                                .setEcsc39(sf("false"))

                                                .setContracttype(simpleQualifier("UNKNOWN", "wt:contractTypes"))

                                                .addFundingtree(

                                                        sf("<fundingtree><funder><id>wt__________::WT</id><shortname>WT</shortname><name>Wellcome Trust</name></funder><funding_level_0><id>wt__________::WT::UNKNOWN</id><description>UNKNOWN</description><name>UNKNOWN</name><class>wt:fundingStream</class><parent/></funding_level_0></fundingtree>"))

                                                .addFundingtree(

                                                        sf("<fundingtree><funder><id>wt__________::WT</id><shortname>WT</shortname><name>Wellcome Trust</name></funder><funding_level_0><id>wt__________::WT::Technology Transfer</id><description>Technology Transfer</description><name>Technology Transfer</name><class>wt:fundingStream</class><parent/></funding_level_0></fundingtree>"))))

                .build();

    }

}

package eu.dnetlib.data.mapreduce.util;

import static org.junit.Assert.assertTrue;

import org.junit.Before;

import org.junit.Test;

import eu.dnetlib.data.proto.FieldTypeProtos.Qualifier;

public class LicenseComparatorTest {

	private LicenseComparator lc;

	@Before

	public void setUp() {

		lc = new LicenseComparator();

	}

	@Test

	public void test() {

		assertTrue(lc.compare(qualifier("OPEN"), qualifier("OPEN")) == 0);

		assertTrue(lc.compare(qualifier("OPEN"), qualifier("EMBARGO")) < 0);

		assertTrue(lc.compare(qualifier("OPEN"), qualifier("RESTRICTED")) < 0);

		assertTrue(lc.compare(qualifier("OPEN"), qualifier("CLOSED")) < 0);

		assertTrue(lc.compare(qualifier("OPEN"), qualifier("UNKNOWN")) < 0);

		assertTrue(lc.compare(qualifier("EMBARGO"), qualifier("OPEN")) > 0);

		assertTrue(lc.compare(qualifier("EMBARGO"), qualifier("EMBARGO")) == 0);

		assertTrue(lc.compare(qualifier("EMBARGO"), qualifier("RESTRICTED")) < 0);

		assertTrue(lc.compare(qualifier("EMBARGO"), qualifier("CLOSED")) < 0);

		assertTrue(lc.compare(qualifier("EMBARGO"), qualifier("UNKNOWN")) < 0);

		assertTrue(lc.compare(qualifier("RESTRICTED"), qualifier("OPEN")) > 0);

		assertTrue(lc.compare(qualifier("RESTRICTED"), qualifier("EMBARGO")) > 0);

		assertTrue(lc.compare(qualifier("RESTRICTED"), qualifier("RESTRICTED")) == 0);

		assertTrue(lc.compare(qualifier("RESTRICTED"), qualifier("CLOSED")) < 0);

		assertTrue(lc.compare(qualifier("RESTRICTED"), qualifier("UNKNOWN")) < 0);

		assertTrue(lc.compare(qualifier("CLOSED"), qualifier("OPEN")) > 0);

		assertTrue(lc.compare(qualifier("CLOSED"), qualifier("EMBARGO")) > 0);

		assertTrue(lc.compare(qualifier("CLOSED"), qualifier("RESTRICTED")) > 0);

		assertTrue(lc.compare(qualifier("CLOSED"), qualifier("CLOSED")) == 0);

		assertTrue(lc.compare(qualifier("CLOSED"), qualifier("UNKNOWN")) < 0);

		assertTrue(lc.compare(qualifier("UNKNOWN"), qualifier("OPEN")) > 0);

		assertTrue(lc.compare(qualifier("UNKNOWN"), qualifier("EMBARGO")) > 0);

		assertTrue(lc.compare(qualifier("UNKNOWN"), qualifier("RESTRICTED")) > 0);

		assertTrue(lc.compare(qualifier("UNKNOWN"), qualifier("CLOSED")) > 0);

		assertTrue(lc.compare(qualifier("UNKNOWN"), qualifier("UNKNOWN")) == 0);

	}

	private Qualifier qualifier(String id) {

		return Qualifier.newBuilder().setClassid(id).build();

	}

}

package eu.dnetlib.data.transform;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import com.googlecode.protobuf.format.JsonFormat;

import eu.dnetlib.data.graph.model.DNGFRowKeyDecoder;

import eu.dnetlib.data.mapreduce.util.dao.HBaseTableDAO;

import eu.dnetlib.data.proto.DNGFProtos.DNGF;

import eu.dnetlib.data.proto.TypeProtos.Type;

import eu.dnetlib.data.proto.dli.Scholix2ObjectProtos;

import eu.dnetlib.dli.proto.DNGF2Scholix2Converter;

import eu.dnetlib.dli.proto.DNGFDLISummaryConverter;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.junit.Before;

import org.junit.Test;

import java.util.Iterator;

import java.util.List;

import java.util.Map;

import java.util.Map.Entry;

import static eu.dnetlib.data.graph.utils.RelDescriptor.QUALIFIER_SEPARATOR;

import static eu.dnetlib.data.proto.dli.Scholix2ObjectProtos.*;

/**

 * Created by sandro on 2/13/17.

 */

public class DLIConvertertTest extends AbstractTransformerTest {

	private static final Log log = LogFactory.getLog(DLIConvertertTest.class);

	private Ontologies ontologies;

    @Before

    public void setUp() throws Exception {

        factory = new XsltRowTransformerFactory();

        ontologies = OntologyLoader.loadOntologiesFromCp();

    }

    @Test

    public void testLinkPangaeaDLI() throws Exception {

        final List<Row> rows = Lists.newArrayList();