D-Net

package eu.dnetlib.data.mapreduce.hbase.oai;

import java.io.IOException;

import java.text.ParseException;

import java.util.Date;

import com.google.common.collect.ArrayListMultimap;

import com.google.common.collect.Lists;

import com.google.common.collect.Multimap;

import com.mongodb.DBObject;

import com.mongodb.client.MongoCollection;

import eu.dnetlib.data.mapreduce.hbase.oai.config.OAIConfiguration;

import eu.dnetlib.data.mapreduce.hbase.oai.config.OAIConfigurationReader;

import eu.dnetlib.data.mapreduce.hbase.oai.config.OAIConfigurationStringReader;

import eu.dnetlib.data.mapreduce.hbase.oai.utils.MongoSetCollection;

import eu.dnetlib.data.mapreduce.hbase.oai.utils.RecordFieldsExtractor;

import org.apache.commons.io.IOUtils;

import org.apache.hadoop.mapreduce.Counter;

import org.dom4j.DocumentException;

import org.junit.Before;

import org.junit.Test;

import org.mockito.Matchers;

import org.mockito.Mock;

import org.mockito.Mockito;

import org.mockito.MockitoAnnotations;

import org.springframework.core.io.ClassPathResource;

import org.springframework.core.io.Resource;

import static org.junit.Assert.*;

public class OAIFeedMapperTest {

	private OaiFeedMapper oaiFeedMapper;

	/** File path where to find the XML configuration profile **/

	private String pathToProfile = "eu/dnetlib/data/mapreduce/hbase/oai/config/OAIPublisherConfiguration-1.xml";

	private OAIConfigurationStringReader oaiConfigurationReader;

	private OAIConfiguration oaiConfiguration;

	@Mock

	private MongoSetCollection mongoSetCollection;

	@Mock

	private MongoCollection<DBObject> discardedCollection;

	@SuppressWarnings("rawtypes")

	@Mock

	private org.apache.hadoop.mapreduce.Mapper.Context context;

	@Mock

	private Counter counter;

	private Date feedDate;

	private String objId1 = "oai:dnet:openaire____::2fa6b215ace86e409dde3ba4b2a6b504";

	private String goodRecord = "<?xml version=\"1.0\"?>\n<record>\n  <result xmlns:dc=\"http://purl.org/dc/elements/1.1/\"\n    xmlns:dnet=\"eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions\"\n    xmlns:dr=\"http://www.driver-repository.eu/namespace/dr\"\n    xmlns:dri=\"http://www.driver-repository.eu/namespace/dri\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">\n    <header>\n      <dri:objIdentifier>openaire____::2fa6b215ace86e409dde3ba4b2a6b504</dri:objIdentifier>\n      <dri:repositoryId/>\n      <dri:dateOfCollection>2013-10-09</dri:dateOfCollection>\n  <dri:dateOfTransformation>2013-10-09</dri:dateOfTransformation>  </header>\n    <metadata>\n      <oaf:entity xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" \n\t\t    xmlns:oaf=\"http://namespace.openaire.eu/oaf\" \n\t\t    xsi:schemaLocation=\"http://namespace.openaire.eu/oaf http://www.openaire.eu/schema/0.1/oaf-0.1.xsd\">\n\t\t<oaf:datasource>\n\t\t\t<officialname>The Internet Journal of Orthopedic Surgery</officialname><englishname>The Internet Journal of Orthopedic Surgery</englishname><websiteurl>http://www.ispub.com/journal/the-internet-journal-of-orthopedic-surgery/</websiteurl><accessinfopackage/><namespaceprefix>issn15312968</namespaceprefix><datasourcetypeui classid=\"pubsrepository::journal\" classname=\"pubsrepository::journal\" schemeid=\"dnet:datasource_typologies\" schemename=\"dnet:datasource_typologies\"/><datasourcetype classid=\"pubsrepository::journal\" classname=\"pubsrepository::journal\" schemeid=\"dnet:datasource_typologies\" schemename=\"dnet:datasource_typologies\"/><openairecompatibility classid=\"notCompatible\" classname=\"notCompatible\" schemeid=\"dnet:compatibilityLevel\" schemename=\"dnet:compatibilityLevel\"/><latitude>0.0</latitude><longitude>0.0</longitude><subjects/><policies name=\"\" id=\"\"/><logourl/><contactemail/><dateofvalidation/><description/><odnumberofitems/><odnumberofitemsdate/><odpolicies/><odlanguages/><odcontenttypes/><releasestartdate/><releaseenddate/><missionstatementurl/><dataprovider>false</dataprovider><serviceprovider>false</serviceprovider><databaseaccesstype/><datauploadtype/><databaseaccessrestriction/><datauploadrestriction/><versioning>false</versioning><citationguidelineurl/><qualitymanagementkind/><pidsystems/><certificates/><originalId>openaire____::issn15312968</originalId><collectedfrom name=\"DOAJ-Articles\" id=\"driver______::bee53aa31dc2cbb538c10c2b65fa5824\"/><pid/><datainfo><inferred>false</inferred><deletedbyinference>false</deletedbyinference><trust>0.9</trust><inferenceprovenance/><provenanceaction classid=\"UNKNOWN\" classname=\"UNKNOWN\" schemeid=\"dnet:provenanceActions\" schemename=\"dnet:provenanceActions\"/></datainfo>\n\t\t  <rels>\n\t\t  </rels>\n\t\t  <children>\n\t\t  </children>\n\t\t</oaf:datasource>\n      </oaf:entity>\n    </metadata>\n  </result>\n</record>";

	private String dedupedRecord = "dedupedRecord.xml";

	private String representativeRecord = "representativeRecord.xml";

	@Before

	public void setUp() throws Exception {

		MockitoAnnotations.initMocks(this);

		oaiFeedMapper = new OaiFeedMapper();

		Resource resource = new ClassPathResource(pathToProfile);

		// setting up the parser and the profile as a string

		String configurationProfile = IOUtils.toString(resource.getInputStream());

		System.out.println("oaiConfiguration:\n" + configurationProfile);

		oaiConfigurationReader = new OAIConfigurationStringReader(configurationProfile);

		oaiConfiguration = oaiConfigurationReader.getOaiConfiguration();

		//String feedDateString = DateUtils.now_ISO8601();

		String feedDateString = "2017-12-18T12:00:04+00:00";

		try {

			feedDate = org.apache.commons.lang.time.DateUtils.parseDate(

					feedDateString,

					new String[]{ "yyyy-MM-dd'T'HH:mm:ssXXX", "yyyy-MM-dd'T'HH:mm:ssZ" });

		} catch (ParseException e) {

			e.printStackTrace(System.err);

			throw new RuntimeException(e);

		}

		oaiFeedMapper.setFeedDate(feedDate);

		oaiFeedMapper.setMongoSetCollection(mongoSetCollection);

		oaiFeedMapper.setOaiConfiguration(oaiConfiguration);

		oaiFeedMapper.setOaiConfigurationReader(oaiConfigurationReader);

		oaiFeedMapper.setDiscardedCollection(discardedCollection);

		oaiFeedMapper.setSkipDuplicates(true);

		oaiFeedMapper.setDuplicateXPath("//entity//datainfo/deletedbyinference[./text() = 'true']");

		Mockito.when(mongoSetCollection.normalizeSetSpec(Matchers.anyString())).thenReturn("aNormalisedSetName");

		Mockito.doNothing().when(counter).increment(Matchers.anyLong());

		Mockito.when(context.getCounter(Matchers.anyString(), Matchers.anyString())).thenReturn(counter);

		//TODO: fix mock

		//Mockito.when(discardedCollection.insertOne((DBObject) Matchers.any());).thenReturn(null);

	}

	@Test

	public void testCreateBasicObject() throws DocumentException, IOException, InterruptedException {

		RecordFieldsExtractor parser = new RecordFieldsExtractor(Lists.newArrayList(oaiConfiguration.getFieldsFor("oaf", "index", "openaire")));

		Multimap<String, String> parsedRecord = parser.extractFields(goodRecord, oaiConfiguration.getEnrichmentXPathsFor("oaf", "index", "openaire"));

		DBObject obj = oaiFeedMapper.createBasicObject(objId1, goodRecord, parsedRecord, context);

		// NOTE that LAST_COLLECTION_DATE_FIELD, DATESTAMP_FIELD,UPDATED_FIELD are not set by the method we are testing, but by the caller

		// method (handleRecord) because they values to set depend on the record status (NEW|UPDATED|UNCHANGED)

		System.out.println(obj);

		assertNotNull(obj);

	}

	@Test

	public void testCreateBasicObjectRep() throws DocumentException, IOException, InterruptedException {

		RecordFieldsExtractor parser = new RecordFieldsExtractor(Lists.newArrayList(oaiConfiguration.getFieldsFor("oaf", "index", "openaire")));

		String repRecordString = IOUtils.toString(this.getClass().getResourceAsStream(representativeRecord));

		Multimap<String, String> parsedRecord = parser.extractFields(repRecordString, oaiConfiguration.getEnrichmentXPathsFor("oaf", "index", "openaire"));

		DBObject obj = oaiFeedMapper.createBasicObject(objId1, repRecordString, parsedRecord, context);

		// NOTE that LAST_COLLECTION_DATE_FIELD, DATESTAMP_FIELD,UPDATED_FIELD are not set by the method we are testing, but by the caller

		// method (handleRecord) because they values to set depend on the record status (NEW|UPDATED|UNCHANGED)

		System.out.println(obj);

		assertNotNull(obj);

	}

	@Test

	public void testParseDeduplicated() throws IOException {

		RecordFieldsExtractor parser = new RecordFieldsExtractor(Lists.newArrayList(oaiConfiguration.getFieldsFor("oaf", "index", "openaire")));

		String dedupedRecordString = IOUtils.toString(this.getClass().getResourceAsStream(dedupedRecord));

		parser.setSkipDuplicates(true);

		parser.setDuplicateXPath("//*[local-name()='entity']//*[local-name()='datainfo']/*[local-name()='deletedbyinference'][./text() = 'true']");

		Multimap<String, String> parsedRecord = parser.extractFields(dedupedRecordString, oaiConfiguration.getEnrichmentXPathsFor("oaf", "index", "openaire"));

		assertFalse(oaiFeedMapper.checkRecordFields(parsedRecord, context, "x", dedupedRecordString));

	}

	@Test

	public void testParseDeduplicated2() throws IOException {

		RecordFieldsExtractor parser = new RecordFieldsExtractor(Lists.newArrayList(oaiConfiguration.getFieldsFor("oaf", "index", "openaire")));

		String dedupedRecordString = IOUtils.toString(this.getClass().getResourceAsStream(dedupedRecord));

		parser.setSkipDuplicates(true);

		parser.setDuplicateXPath("//x");

		Multimap<String, String> parsedRecord = parser.extractFields(dedupedRecordString, oaiConfiguration.getEnrichmentXPathsFor("oaf", "index", "openaire"));

		assertTrue(oaiFeedMapper.checkRecordFields(parsedRecord, context, "x", dedupedRecordString));

	}

	@Test

	public void testCheckRecordFieldsDeduplicated() {

		Multimap<String, String> recordProps = ArrayListMultimap.create();

		recordProps.put("duplicate", "true");

		assertFalse(oaiFeedMapper.checkRecordFields(recordProps, context, objId1, goodRecord));

	}

	@Test

	public void testCheckRecordFieldsNotDeduplicated() {

		Multimap<String, String> recordProps = ArrayListMultimap.create();

		recordProps.put("duplicate", "false");

		recordProps.put(OAIConfigurationReader.ID_FIELD, "10|xxxx");

		assertTrue(oaiFeedMapper.checkRecordFields(recordProps, context, objId1, goodRecord));

	}

	@Test

	public void testCheckRecordFieldsEmpty() {

		Multimap<String, String> recordProps = ArrayListMultimap.create();

		assertFalse(oaiFeedMapper.checkRecordFields(recordProps, context, objId1, goodRecord));

	}

	@Test

	public void testCheckRecordFieldsNull() {

		Multimap<String, String> recordProps = null;

		assertFalse(oaiFeedMapper.checkRecordFields(recordProps, context, objId1, goodRecord));

	}

	@Test

	public void testTransformationDate(){

		RecordFieldsExtractor parser = new RecordFieldsExtractor(Lists.newArrayList(oaiConfiguration.getFieldsFor("oaf", "index", "openaire")));

		parser.setSkipDuplicates(true);

		parser.setDuplicateXPath("//*[local-name()='entity']//*[local-name()='datainfo']/*[local-name()='deletedbyinference'][./text() = 'true']");

		Multimap<String, String> parsedRecord = parser.extractFields(goodRecord, oaiConfiguration.getEnrichmentXPathsFor("oaf", "index", "openaire"));

		System.out.println(parsedRecord);

		assertTrue(parsedRecord.containsKey("dateOfTransformation"));

	}

}