D-Net

{

"type_source": "SVN",

"goal": "package -U source:jar",

"url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet50/modules/dnet-data-services/branches/saxonHE",

"deploy_repository": "dnet5-snapshots",

"version": "5",

"mail": "alessia.bardi@isti.cnr.it",

"deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet5-snapshots",

"name": "dnet-data-services-saxonHE"

}

package eu.dnetlib.enabling.tools.blackboard;

import eu.dnetlib.data.mdstore.modular.action.MDStoreActions;

public class SampleCreateAction implements BlackboardServerAction<MDStoreActions> {

	@Override

	public void execute(BlackboardServerHandler handler, BlackboardJob job) {

		handler.done(job);

	}

}

package eu.dnetlib.enabling.tools.blackboard;

import org.springframework.beans.factory.FactoryBean;

import static org.mockito.Mockito.mock;

/**

 * Created by sandro on 3/22/16.

 */

public class MockBeanFactory implements FactoryBean<BlackboardServerHandler> {

	@Override

	public BlackboardServerHandler getObject() throws Exception {

		return mock(BlackboardServerHandler.class);

	}

	@Override

	public Class<BlackboardServerHandler> getObjectType() {

		return BlackboardServerHandler.class;

	}

	@Override

	public boolean isSingleton() {

		return true;

	}

}

package eu.dnetlib.enabling.tools.blackboard;

import eu.dnetlib.data.mdstore.modular.action.MDStoreActions;

import org.junit.Before;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.test.context.ContextConfiguration;

import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import static org.junit.Assert.assertNotNull;

import static org.mockito.Matchers.anyObject;

import static org.mockito.Matchers.eq;

import static org.mockito.Mockito.*;

@RunWith(SpringJUnit4ClassRunner.class)

@ContextConfiguration

public class BlackboardServerActionExecutorTest {

	@Autowired

	public BlackboardServerHandler blackboardHandler;

	@Autowired

	public transient BlackboardServerActionExecutor<MDStoreActions> executor;

	@Before

	public void setUp() throws Exception {

	}

	@Test

	public void testExecutor() {

		assertNotNull(executor);

		BlackboardJob job = mock(BlackboardJob.class);

		when(job.getAction()).thenReturn("CREATE");

		executor.execute(job);

		verify(blackboardHandler).done(eq(job));

	}

	@Test

	public void testExecutorUnimplemented() {

		assertNotNull(executor);

		BlackboardJob job = mock(BlackboardJob.class);

		when(job.getAction()).thenReturn("DELETE");

		executor.execute(job);

		verify(blackboardHandler).failed(eq(job), (Throwable) anyObject());

	}

}

package eu.dnetlib.data.collector.plugins;

import com.google.common.collect.Lists;

import eu.dnetlib.rmi.data.CollectorServiceException;

import eu.dnetlib.rmi.data.InterfaceDescriptor;

import org.junit.Assert;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

/**

 * Created by Alessia Bardi on 10/08/2017.

 *

 * @author Alessia Bardi

 */

@Ignore

public class HttpSimpleCollectorPluginTest {

	private InterfaceDescriptor apiDescriptor;

	private HttpSimpleCollectorPlugin plugin;

	@Before

	public void prepare(){

		plugin = new HttpSimpleCollectorPlugin();

		apiDescriptor = new InterfaceDescriptor();

		apiDescriptor.setBaseUrl("http://data.d4science.org/em1EemhBdUZ0bjNGTWJNNjlxVDltcm9acDFmMHlBSVVHbWJQNStIS0N6Yz0");

	}

	@Test

	public void test() throws CollectorServiceException {

	Iterable<String> res = plugin.collect(apiDescriptor, null, null);

	Assert.assertTrue(Lists.newArrayList(res).size() == 1);

	for(String r : res){

		System.out.println(r);

		Assert.assertNotNull(r);

	}

	}

}

package eu.dnetlib.data.collector.plugins.sftp;

import eu.dnetlib.rmi.data.CollectorServiceException;

import eu.dnetlib.rmi.data.InterfaceDescriptor;

import org.junit.Assert;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.test.context.ContextConfiguration;

import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

/**

 * Created by alessia on 16/03/17.

 */

@RunWith(SpringJUnit4ClassRunner.class)

@ContextConfiguration

@Ignore

public class SftpPubKeyCollectorPluginTest {

	@Autowired

	private  SftpPubKeyCollectorPlugin plugin;

	private InterfaceDescriptor apiDescriptor;

	private int max_records = 10;

	@Before

	public void prepare(){

		apiDescriptor = new InterfaceDescriptor();

		apiDescriptor.setBaseUrl("sftp://ariadne2.isti.cnr.it/../../data/transform/old/acdm_correct");

		apiDescriptor.setProtocol("sftp");

		apiDescriptor.setId("testSFTP");

		apiDescriptor.getParams().put("recursive", "true");

		apiDescriptor.getParams().put("extensions", "xml");

		apiDescriptor.getParams().put("username", "alessia");

	}

	@Test

	public void testWithAriadneServer() throws CollectorServiceException {

		Iterable<String> res = plugin.collect(apiDescriptor, null, null);

		Assert.assertNotNull(res);

		int i =0;

		for(String r : res){

			i++;

			Assert.assertNotNull(r);

			if(i > max_records) break;

		}

	}

	@Test

	public void testIncrementalWithAriadneServer() throws CollectorServiceException {

		Iterable<String> res = plugin.collect(apiDescriptor, "2017-06-06", null);

		Assert.assertNotNull(res);

		int i =0;

		for(String r : res){

			i++;

			Assert.assertNotNull(r);

			if(i > max_records) break;

		}

	}

}

package eu.dnetlib.data.cleaner;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertNotNull;

import static org.junit.Assert.assertNull;

import static org.mockito.Matchers.anyString;

import static org.mockito.Mockito.times;

import static org.mockito.Mockito.verify;

import static org.mockito.Mockito.when;

import java.io.StringReader;

import java.util.List;

import org.dom4j.Document;

import org.dom4j.io.SAXReader;

import org.junit.Before;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.Mock;

import org.mockito.runners.MockitoJUnitRunner;

import com.google.common.collect.Lists;

import com.google.common.collect.Sets;

import eu.dnetlib.data.cleaner.VocabularyRule;

import eu.dnetlib.rmi.enabling.ISLookUpService;

@RunWith(MockitoJUnitRunner.class)

public class VocabularyRuleTest {

	private static final String VOCABULARY_NAME_1 = "TEST VOCABULARY 1";

	private static final String VOCABULARY_NAME_2 = "TEST VOCABULARY 2";

	private static final List<String> VOCABULARY = Lists.newArrayList("XXXX|-:-|AAAA", "YYYY|-:-|AAAA", "ZZZZ|-:-|AAAA");

	/**

	 * Class Under Test

	 */

	private VocabularyRule rule;

	@Mock

	private ISLookUpService lookup;

	@Before

	public void setUp() throws Exception {

		when(this.lookup.quickSearchProfile(anyString())).thenReturn(VOCABULARY);

		this.rule = new VocabularyRule(Sets.newHashSet(VOCABULARY_NAME_1, VOCABULARY_NAME_2), this.lookup);

	}

	@Test

	public void testSetup() throws Exception {

		final String xpath = "/a/b";

		this.rule.setXpath(xpath);

		execute("<a><b>XXXX</b></a>");

		verify(this.lookup, times(2)).quickSearchProfile(anyString());

		assertEquals(VOCABULARY.size(), this.rule.getVocabularyTerms().size());

	}

	@Test

	public void testApplyXpathRule() throws Exception {

		final String xpath = "/a/b";

		this.rule.setXpath(xpath);

		final Document doc = execute("<a><b>XXXX</b></a>");

		assertEquals("AAAA", doc.valueOf(xpath));

		assertNull(this.rule.verifyValue("AAAA"));

		assertNotNull(this.rule.verifyValue("XXXX"));

	}

	@Test

	public void testApplyXpathRule_2() throws Exception {

		final String xpath = "/a/b";

		this.rule.setXpath(xpath);

		final Document doc = execute("<a><b>XXXX</b></a>");

		assertEquals("AAAA", doc.valueOf(xpath));

		assertNull(this.rule.verifyValue("AAAA"));

		assertNotNull(this.rule.verifyValue("XXXX"));

	}

	@Test

	public void testApplyXpathRule_3() throws Exception {

		final String xpath = "/a/b";

		this.rule.setXpath(xpath);

		final Document doc = execute("<a><b>XXXX</b></a>");

		assertEquals("AAAA", doc.valueOf(xpath));

	}

	@Test

	public void testApplyXpathRule_attr() throws Exception {

		final String xpath = "/a/b/@value";

		this.rule.setXpath(xpath);

		final Document doc = execute("<a><b value='XXXX' /></a>");

		assertEquals("AAAA", doc.valueOf(xpath));

		assertNull(this.rule.verifyValue("AAAA"));

		assertNotNull(this.rule.verifyValue("XXXX"));

	}

	@Test

	public void testApplyXpathRule_with_spaces() throws Exception {

		final String xpath = "/a/b";

		this.rule.setXpath(xpath);

		final Document doc = execute("<a><b>  XXXX  </b></a>");

		assertEquals("AAAA", doc.valueOf(xpath));

		assertNull(this.rule.verifyValue("AAAA"));

		assertNotNull(this.rule.verifyValue("  XXXX  "));

	}

	@Test

	public void testApplyXpathRule_case() throws Exception {

		final String xpath = "/a/b";

		this.rule.setXpath(xpath);

		final Document doc = execute("<a><b>Xxxx</b></a>");

		assertEquals("AAAA", doc.valueOf(xpath));

		assertNull(this.rule.verifyValue("AAAA"));

		assertNotNull(this.rule.verifyValue("Xxxx"));

	}

	private Document execute(final String xml) throws Exception {

		final SAXReader reader = new SAXReader();

		final Document doc = reader.read(new StringReader(xml));

		System.out.println("BEFORE:\n" + doc.asXML() + "\n");

		this.rule.applyXpathRule(doc);

		System.out.println("AFTER:\n" + doc.asXML() + "\n");

		System.out.println("-----------------------------\n");

		return doc;

	}

}

package eu.dnetlib.data.cleaner;

import static org.junit.Assert.assertFalse;

import static org.junit.Assert.assertTrue;

import java.util.HashMap;

import java.util.Map;

import org.junit.Before;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.runners.MockitoJUnitRunner;

import com.google.common.collect.Lists;

import eu.dnetlib.data.cleaner.CleaningRule;

import eu.dnetlib.data.cleaner.XPATHCleaningRule;

import eu.dnetlib.rmi.data.CleanerException;

@RunWith(MockitoJUnitRunner.class)

public class XMLCleaningRuleTest {

	private static final String INPUT_VALID = "<record>" + "	<header />" + "	<metadata>" + "		<a>HELLO</a>" + "	</metadata>" + "</record>";

	private static final String INPUT_INVALID = "<record>" + "	<header />" + "	<metadata>" + "		<a>GOOD BYE</a>" + "	</metadata>" + "</record>";

	/**

	 * Class under test.

	 */

	private CleaningRule xmlRule;

	private final XPATHCleaningRule mockXpathRule = new XPATHCleaningRule() {

		@Override

		protected Map<String, String> verifyValue(final String value) throws CleanerException {

			if (value.equals("CIAO")) { return null; }

			final Map<String, String> err = new HashMap<String, String>();

			err.put("term", value);

			return err;

		}

		@Override

		protected String calculateNewValue(final String oldValue) throws CleanerException {

			if (oldValue.equals("HELLO")) { return "CIAO"; }

			return oldValue;

		}

	};;

	@Before

	public void setUp() throws Exception {

		this.xmlRule = new CleaningRule();

		this.mockXpathRule.setStrict(true);

		this.mockXpathRule.setXpath("//a");

		this.xmlRule.setXpathRules(Lists.newArrayList(this.mockXpathRule));

	}

	@Test

	public void testEvaluate_valid() {

		final String s = this.xmlRule.apply(INPUT_VALID);

		assertTrue(s.contains("CIAO"));

		assertFalse(s.contains("invalid"));

	}

	@Test

	public void testEvaluate_invalid() {

		final String s = this.xmlRule.apply(INPUT_INVALID);

		System.out.println(s);

		assertFalse(s.contains("CIAO"));

		assertTrue(s.contains("invalid"));

	}

}

package eu.dnetlib.data.cleaner;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertNotNull;

import static org.mockito.Mockito.verify;

import static org.mockito.Mockito.when;

import org.junit.Before;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.Mock;

import org.mockito.runners.MockitoJUnitRunner;

import eu.dnetlib.data.cleaner.CleanerServiceImpl;

import eu.dnetlib.data.cleaner.CleaningRule;

import eu.dnetlib.data.cleaner.CleaningRuleFactory;

import eu.dnetlib.enabling.resultset.factory.ResultSetFactory;

import eu.dnetlib.rmi.common.ResultSet;

import eu.dnetlib.rmi.data.CleanerException;

@RunWith(MockitoJUnitRunner.class)

public class CleanerServiceImplTest {

	/**

	 * Class under test.

	 */

	private CleanerServiceImpl service;

	@Mock

	private CleaningRuleFactory cleaningRuleFactory;

	@Mock

	private ResultSetFactory resultSetFactory;

	@Mock

	private CleaningRule cleaningRule;

	@Mock

	private ResultSet<String> rs_IN;

	@Mock

	private ResultSet<String> rs_OUT;

	private static final String RULE_ID = "RULE_01";

	@Before

	public void setUp() throws Exception {

		when(this.cleaningRuleFactory.obtainCleaningRule(RULE_ID)).thenReturn(this.cleaningRule);

		when(this.resultSetFactory.map(this.rs_IN, String.class, this.cleaningRule)).thenReturn(this.rs_OUT);

		this.service = new CleanerServiceImpl();

		this.service.setCleaningRuleFactory(this.cleaningRuleFactory);

		this.service.setResultSetFactory(this.resultSetFactory);

	}

	@Test

	public void testClean() throws CleanerException {

		final ResultSet<String> rs = this.service.clean(this.rs_IN, RULE_ID);

		assertNotNull(rs);

		assertEquals(this.rs_OUT, rs);

		verify(this.cleaningRuleFactory).obtainCleaningRule(RULE_ID);

	}

	@Test(expected = CleanerException.class)

	public void testClean_null_1() throws CleanerException {

		this.service.clean(this.rs_IN, null);

	}

	@Test(expected = CleanerException.class)

	public void testClean_null_2() throws CleanerException {

		this.service.clean(null, RULE_ID);

	}

}

package eu.dnetlib.data.cleaner;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertTrue;

import java.io.StringReader;

import java.util.List;

import org.dom4j.Document;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.runners.MockitoJUnitRunner;

import eu.dnetlib.data.cleaner.GroovyRule;

@RunWith(MockitoJUnitRunner.class)

public class GroovyRuleTest {

	@Test

	public void testApplyXpathRule_simple_constant() throws Exception {

		final GroovyRule rule = new GroovyRule("'YYYY'");

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute(rule, "<a><b>XXXX</b></a>");

		assertEquals("YYYY", doc.valueOf(xpath));

	}

	@Test

	public void testApplyXpathRule_simple_regex() throws Exception {

		final GroovyRule rule = new GroovyRule("(input =~ /X/).replaceAll('Y')");

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute(rule, "<a><b>aXaXa</b></a>");

		assertEquals("aYaYa", doc.valueOf(xpath));

	}

	@Test

	public void testApplyXpathRule_simple_upper() throws Exception {

		final GroovyRule rule = new GroovyRule("input.toUpperCase()");

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute(rule, "<a><b>xyz</b></a>");

		assertEquals("XYZ", doc.valueOf(xpath));

	}

	@Test

	public void testApplyXpathRule_multi() throws Exception {

		final GroovyRule rule = new GroovyRule("'Y'");

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute(rule, "<a><b>X</b><b>X</b><b>X</b></a>");

		final List<?> list = doc.selectNodes(xpath);

		assertEquals(3, list.size());

		for (final Object o : list) {

			assertEquals("Y", ((Node) o).getText());

		}

	}

	@Test

	public void testApplyXpathRule_singleAttr() throws Exception {

		final GroovyRule rule = new GroovyRule("'BBBB'");

		final String xpath = "/a/b/@value";

		rule.setXpath(xpath);

		final Document doc = execute(rule, "<a><b value='AAAA'>XXXX</b></a>");

		assertEquals("BBBB", doc.valueOf(xpath));

		assertEquals("XXXX", doc.valueOf("/a/b"));

	}

	@Test

	public void testApplyXpathRule_multiAttr() throws Exception {

		final GroovyRule rule = new GroovyRule("'B'");

		final String xpath = "/a/b/@value";

		rule.setXpath(xpath);

		final Document doc = execute(rule, "<a><b value='a' /><b value='b' /><b value='c' /></a>");

		final List<?> list = doc.selectNodes(xpath);

		assertEquals(3, list.size());

		for (final Object o : list) {

			assertEquals("B", ((Node) o).getText());

		}

	}

	@Test

	public void testApplyXpathRule_complex() throws Exception {

		final GroovyRule rule = new GroovyRule("'B'");

		final String xpath = "/a/b";

		rule.setXpath(xpath);

		final Document doc = execute(rule, "<a><b>X<c>C</c></b></a>");

		assertTrue(doc.valueOf(xpath).contains("B"));

		assertEquals("C", doc.valueOf("/a/b/c"));

	}

	private Document execute(final GroovyRule rule, final String xml) throws Exception {

		final SAXReader reader = new SAXReader();

		final Document doc = reader.read(new StringReader(xml));

		System.out.println("BEFORE:\n" + doc.asXML() + "\n");

		rule.applyXpathRule(doc);

		System.out.println("AFTER:\n" + doc.asXML() + "\n");

		System.out.println("-----------------------------\n");

		return doc;

	}

}

package eu.dnetlib.data.mdstore.modular.mongodb;

import com.mongodb.BasicDBObject;

import com.mongodb.DBObject;

import com.mongodb.client.MongoCollection;

import com.mongodb.client.MongoDatabase;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDBStatus;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;

import eu.dnetlib.rmi.data.MDStoreServiceException;

import org.junit.After;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.test.context.ContextConfiguration;

import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import static org.junit.Assert.assertEquals;

@Ignore

@RunWith(SpringJUnit4ClassRunner.class)

@ContextConfiguration(classes = ConfigurationTestConfig.class)

public class MDStoreDaoImplTest {

	@Autowired

	private MDStoreDao dao;

	@Autowired

	private MongoDatabase db;

	@Autowired

	private MDStoreTransactionManager manager;

	@After

	public void tearDown() throws MDStoreServiceException {

		dao.deleteMDStore("1");

		dao.deleteMDStore("2");

		dao.deleteMDStore("3");

		dao.deleteMDStore("4");

		dao.deleteMDStore("5");

		dao.deleteMDStore("6");

	}

	@Before

	public void setUp() throws Exception {

		dao.createMDStore("1", "F", "I", "L");

		dao.createMDStore("2", "F", "I", "L");

		dao.createMDStore("3", "F", "I", "L");

		dao.createMDStore("4", "F", "I", "L");

		dao.createMDStore("5", "F1", "I", "L");

		dao.createMDStore("6", "F1", "I", "L");

		final MongoCollection<DBObject> metadata = db.getCollection(MDStoreDaoImpl.METADATA_NAME, DBObject.class);

		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "1"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));

		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "2"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));

		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "3"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));

		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "4"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));

		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "5"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));

		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "6"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));

	}

	@Test

	public void test() throws MDStoreServiceException {

		assertEquals(40, dao.getSumOfSizes("F", "L", "I"));

		assertEquals(20, dao.getSumOfSizes("F1", "L", "I"));

		assertEquals(0, dao.getSumOfSizes("F_0", "L", "I"));

	}

	@Test

	public void getDBStatusTest() {

		final MDStoreDBStatus dbStatus = dao.getDBStatus();

		System.out.println(dbStatus);

	}

}

package eu.dnetlib.data.mdstore.modular.mongodb;

import java.io.File;

import java.io.FileInputStream;

import java.io.IOException;

import java.util.Iterator;

import java.util.UUID;

import com.mongodb.DBObject;

import com.mongodb.client.MongoDatabase;

import eu.dnetlib.data.mdstore.modular.RecordParserFactory;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.rmi.data.MDStoreServiceException;

import org.apache.commons.io.IOUtils;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.core.io.ClassPathResource;

import org.springframework.test.context.ContextConfiguration;

import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

@Ignore

@RunWith(SpringJUnit4ClassRunner.class)

@ContextConfiguration(classes = ConfigurationTestConfig.class)

public class FeedSpeedTest {

	private static final int N_RECORDS = 68271;

	@Autowired

	private MongoDatabase db;

	@Autowired

	private MDStoreDao dao;

	@Autowired

	private RecordParserFactory recordParserfactory;

	@Before

	public void setup() throws MDStoreServiceException {

		dao.createMDStore("speed_test", "testFormat", "testInterpretation", "testLayout");

	}

	@Test

	public void testSpeedFromFolder() throws IOException {

		Iterable<String> iterable = new Iterable<String>() {

			private int counter = 0;

			private double last = System.currentTimeMillis();

			@Override

			public Iterator<String> iterator() {

				return new Iterator<String>() {

					@Override

					public boolean hasNext() {

						return counter < N_RECORDS;

					}

					@Override

					public String next() {

						if (counter % 10000 == 0) {

							System.out.println("10K records processed in " + (System.currentTimeMillis() - last) / 1000 + " seconds");

							last = System.currentTimeMillis();

						}

						File f = new File(String.format("/var/lib/eagle/content/EDH/HD%06d.xml", counter++));

						if (f.exists()) {

							try {

								FileInputStream fileInputStream = new FileInputStream(f);

								String s = IOUtils.toString(fileInputStream);

								fileInputStream.close();

								return s;

							} catch (Exception e) {

								return null;

							}

						} else {

							try {

								FileInputStream fileInputStream = new FileInputStream(new File("/var/lib/eagle/content/EDH/HD000001.xml"));

								String s = IOUtils.toString(fileInputStream);

								fileInputStream.close();

								return s;

							} catch (Exception e) {

								return null;

							}

						}

					}

					@Override

					public void remove() {

					}

				};

			}

		};

		MongoMDStore mdStore =

				new MongoMDStore(UUID.randomUUID().toString(), db.getCollection("speed_test", DBObject.class), recordParserfactory.newInstance(), true, db);

		mdStore.feed(iterable, false);

	}

	//@Ignore

	@Test

	public void testFeedSpeedFromTemplate() throws MDStoreServiceException, IOException {

		MongoMDStore mdStore =

				new MongoMDStore(UUID.randomUUID().toString(), db.getCollection("speed_test", DBObject.class), recordParserfactory.newInstance(), false, db);

		mdStore.feed(new Iterable<String>() {

			private int counter = 0;

			private double last = System.currentTimeMillis();

			private String templateRecord = IOUtils.toString(new ClassPathResource("/eu/dnetlib/data/mdstore/modular/mongodb/templateRecord.xml")

					.getInputStream());

			@Override

			public Iterator<String> iterator() {

				return new Iterator<String>() {

					@Override

					public boolean hasNext() {

						return counter < N_RECORDS;

					}

					@Override

					public String next() {

						if (counter % 10000 == 0) {

							System.out.println("10K records processed in " + (System.currentTimeMillis() - last) / 1000 + " seconds");

							last = System.currentTimeMillis();

						}

						File f = new File(String.format("/var/lib/eagle/content/EDH/HD%06d.xml", counter++));

						if (f.exists()) {

							try {

								FileInputStream fileInputStream = new FileInputStream(f);

								String s = IOUtils.toString(fileInputStream);

								fileInputStream.close();

								return s;

							} catch (Exception e) {

								return null;

							}

						} else {

							counter++;

							try {

								FileInputStream fileInputStream = new FileInputStream(new File("/var/lib/eagle/content/EDH/HD000009.xml"));

								String s = IOUtils.toString(fileInputStream);

								fileInputStream.close();

								return s;

							} catch (Exception e) {

								return null;

							}

						}

					}

					@Override

					public void remove() {

					}

				};

			}

		}, false);

	}

}

package eu.dnetlib.data.mdstore.modular.mongodb;

import java.net.UnknownHostException;

import com.mongodb.MongoClient;

import com.mongodb.client.MongoDatabase;

import eu.dnetlib.data.mdstore.modular.RecordParserFactory;

import eu.dnetlib.data.mdstore.modular.StreamingRecordParser;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;

import org.springframework.context.annotation.Bean;

import org.springframework.context.annotation.Configuration;

@Configuration

public class ConfigurationTestConfig {

	@Bean

	public MongoDatabase db() throws UnknownHostException {

		final MongoClient mongo = new MongoClient("localhost", 27017);

		return mongo.getDatabase("mdstore_test");

	}

	@Bean

	public MDStoreTransactionManager manager() throws UnknownHostException {

		final MDStoreTransactionManagerImpl manager = new MDStoreTransactionManagerImpl();

		manager.setDb(db());

		return manager;

	}

	@Bean

	public RecordParserFactory recordParserFactory() {

		final RecordParserFactory rpfactory = new RecordParserFactory();

		rpfactory.setParserType(StreamingRecordParser.class);

		return rpfactory;

	}

	@Bean

	public MDStoreDao mdstoreDao() throws UnknownHostException {

		final MDStoreDaoImpl dao = new MDStoreDaoImpl();

		dao.setDb(db());

		dao.setRecordParserFactory(recordParserFactory());

		return dao;

	}

}

package eu.dnetlib.data.mdstore.modular.mongodb;

import java.net.UnknownHostException;

import java.util.ArrayList;

import java.util.Iterator;

import java.util.UUID;

import com.mongodb.DBObject;

import com.mongodb.MongoClient;

import com.mongodb.client.FindIterable;

import com.mongodb.client.MongoCollection;

import com.mongodb.client.MongoDatabase;

import eu.dnetlib.data.mdstore.modular.connector.MDStore;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;

import eu.dnetlib.enabling.resultset.listener.ResultSetListener;

import eu.dnetlib.rmi.data.MDStoreServiceException;

import org.junit.Assert;

import org.junit.Ignore;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.test.context.ContextConfiguration;

import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

// TODO: reimplement tests

@Ignore

@RunWith(SpringJUnit4ClassRunner.class)

@ContextConfiguration(classes = ConfigurationTestConfig.class)

public class MDStoreTransactionManagerTest {

	@Autowired

	private MongoDatabase db;

	@Autowired

	private MDStoreTransactionManager manager;

	@Autowired

	private MDStoreDao dao;

	@Test

	public void testCreateandRetrieve() throws MDStoreServiceException {

		UUID idCreation = UUID.randomUUID();

		db.getCollection("metadataManager").drop();

		((MDStoreTransactionManagerImpl) manager).setManagerTable(null);

		String mdId = idCreation.toString() + "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";

		manager.createMDStore(mdId);

		Assert.assertNotNull(manager.startTransaction(mdId, true));

		Assert.assertNotNull(manager.startTransaction(mdId, true));

		String s = manager.getMDStoreCollection(mdId);

		Assert.assertNotNull(s);

		this.manager.dropMDStore(mdId);

		s = manager.getMDStoreCollection(mdId);

		Assert.assertNull(s);

		db.getCollection("metadataManager").drop();

		((MDStoreTransactionManagerImpl) manager).setManagerTable(null);

	}

	@Test

	public void testReadMdStore() throws MDStoreServiceException {

		UUID idCreation = UUID.randomUUID();

		String mdId = idCreation.toString() + "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";

		manager.createMDStore(mdId);

		Assert.assertNotNull(manager.readMdStore(mdId));

		Assert.assertNotNull(manager.startTransaction(mdId, true));

		Assert.assertNotNull(manager.readMdStore(mdId));

		db.getCollection("metadataManager").drop();

		((MDStoreTransactionManagerImpl) manager).setManagerTable(null);

		db.getCollection("metadataManager").drop();

		((MDStoreTransactionManagerImpl) manager).setManagerTable(null);

	}

	@Test

	public void testCommit() throws MDStoreServiceException {

		UUID idCreation = UUID.randomUUID();

		String mdId = idCreation.toString() + "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";

		manager.createMDStore(mdId);

		String idCurrent = manager.readMdStore(mdId);

		String transaction = manager.startTransaction(mdId, true);

		// Assert.assertTrue(manager.commit(transaction, mdId));

		Assert.assertNotSame(idCurrent, manager.readMdStore(mdId));

	}

	@Ignore

	@Test

	public void testDateTime() throws MDStoreServiceException, UnknownHostException {

		MongoClient mongo = new MongoClient("localhost", 27017);

		MongoDatabase dbinput = mongo.getDatabase("mdstore");

		MongoCollection<DBObject> inputCollection = dbinput.getCollection("70e07e9f-b3bf-4423-8777-b159819e0c6a", DBObject.class);

		Assert.assertNotNull(inputCollection.find().first().get("body"));

		UUID idCreation = UUID.randomUUID();

		String mdId = idCreation.toString() + "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";

		manager.createMDStore(mdId);

		dao.createMDStore(mdId, "a", "a", "a");

		String transId = manager.startTransaction(mdId, true);

		ArrayList<String> data = new ArrayList<String>();

		FindIterable<DBObject> cursor = inputCollection.find();

		Iterator<DBObject> it = cursor.iterator();

		for (int i = 0; i < 1000; i++) {

			data.add((String) it.next().get("body"));

		}

		dao.getMDStore(transId).feed(data, true);

		// manager.commit(transId, mdId);

		cursor = inputCollection.find();

		it = cursor.iterator();

		transId = manager.startTransaction(mdId, false);

		data.clear();

		for (int i = 0; i < 10; i++) {

			data.add(it.next().get("body").toString().replace("oai:pumaoai.isti.cnr.it:", "SUUUCAAA"));

		}

		String currentId = manager.readMdStore(mdId);

		final MDStore newMdstore = dao.getMDStore(currentId);

		new Thread(() -> {

			final ResultSetListener<String> deliver = newMdstore.deliver("", "", null, false);

			for (int i = 0; i < 10; i++) {

				try {

					Thread.sleep(1000);

					System.out.println(deliver.next());

				} catch (Exception e) {

					// TODO Auto-generated catch block

					e.printStackTrace();

				}

			}

		}).start();

		dao.getMDStore(transId).feed(data, true);

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

       xmlns:p="http://www.springframework.org/schema/p" xmlns="http://www.springframework.org/schema/beans"

       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">

	<bean id="blackboardHandler" class="eu.dnetlib.enabling.tools.blackboard.MockBeanFactory"/>

	<bean id="executor"

	      class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerActionExecutor"

	      p:blackboardHandler-ref="blackboardHandler" p:actionType="eu.dnetlib.data.mdstore.modular.action.MDStoreActions"

	      p:incomplete="true">

		<property name="actionMap">

			<map>

				<entry key="CREATE">

					<bean class="eu.dnetlib.enabling.tools.blackboard.SampleCreateAction"/>

				</entry>

			</map>

		</property>

	</bean>

</beans>

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

       xmlns:p="http://www.springframework.org/schema/p" xmlns="http://www.springframework.org/schema/beans"

       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">

	<bean id="sftpPubKeyCollectorPlugin" class="eu.dnetlib.data.collector.plugins.sftp.SftpPubKeyCollectorPlugin"

	      p:sftpIteratorFactory-ref="sftpIteratorFactory">

		<property name="protocolDescriptor">

			<bean class="eu.dnetlib.rmi.data.ProtocolDescriptor" p:name="sftpPubKeyAuthentication">

				<property name="params">

					<list>

						<bean class="eu.dnetlib.rmi.data.ProtocolParameter"

						      p:name="username" />

						<bean class="eu.dnetlib.rmi.data.ProtocolParameter"

						      p:name="recursive" p:type="BOOLEAN"/>

						<bean class="eu.dnetlib.rmi.data.ProtocolParameter"

						      p:name="extensions" p:type="LIST"/>

					</list>

				</property>

			</bean>

		</property>

	</bean>

	<bean id="sftpIteratorFactory" class="eu.dnetlib.data.collector.plugins.sftp.SftpIteratorFactory"/>

	<bean id="propertyConfigurer"

	      class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">

		<property name="properties">

			<props>

				<prop key="collector.sftp.auth.prvKeyPath">~/.ssh/id_rsa</prop>

				<prop key="collector.sftp.auth.passphrase"></prop>

				<prop key="collector.sftp.auth.knownHostsPath">~/.ssh/known_hosts</prop>

			</props>

		</property>

	</bean>

</beans>

<oai:record xmlns:dri="http://www.driver-repository.eu/namespace/dri"

            xmlns:oai="http://www.openarchives.org/OAI/2.0/"

            xmlns="http://namespace.openaire.eu/"

>

	<oai:header>

		<dri:objIdentifier>{id}</dri:objIdentifier>

		<dri:recordIdentifier>entityId:2533216</dri:recordIdentifier>

		<dri:dateOfCollection>2015-01-09T12:15:02.177+01:00</dri:dateOfCollection>

		<dri:repositoryId>8e17a4fe-b77d-406b-8c75-4cfa233092d2_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=</dri:repositoryId>

		<dri:datasourceprefix>DAI</dri:datasourceprefix>

	</oai:header>

	<oai:metadata>

		<eagle:eagleObject xmlns:eagle="http://www.eagle-network.eu/eagle">

			<eagle:recordSourceInfo

					landingPage="http://arachne.uni-koeln.de/entity/2533216" providerName="Arachne">entityId:2533216

			</eagle:recordSourceInfo>

			<eagle:editingInfo>

				<eagle:dateEdited>0000-00-00</eagle:dateEdited>

				<eagle:metadataEditor>unknown</eagle:metadataEditor>

			</eagle:editingInfo>

			<eagle:metadataIpr uri="http://creativecommons.org/licenses/by/3.0/">This file is licensed under the Creative Commons Attribution 3.0 Unported license.</eagle:metadataIpr>

			<eagle:title lang="de">CMS-XII-d009c-3_168111,02.jpg</eagle:title>

			<!-- objekt http://arachne.uni-koeln.de/entity/1160926 -->

			<eagle:description lang="de">Foto von: Siegel CMS XII D009c</eagle:description>

			<eagle:entityType>visual</eagle:entityType>

			<eagle:visualRepresentation>

				<eagle:representationType>image</eagle:representationType>

				<eagle:url>http://arachne.uni-koeln.de/entity/2533216</eagle:url>

				<eagle:thumbnail>http://arachne.dainst.org/data/image/thumbnail/2533216</eagle:thumbnail>

				<eagle:visualRepresentationIpr uri="http://creativecommons.org/licenses/by/3.0/">This file is licensed under the Creative Commons Attribution 3.0 Unported license.</eagle:visualRepresentationIpr>

				<eagle:format>jpg</eagle:format>

				<!-- objekt http://arachne.uni-koeln.de/entity/1160926 -->

				<eagle:hasArtifact>

					<eagle:artifactTitle>Siegel CMS XII D009c</eagle:artifactTitle>

					<eagle:objectType>Dreiseitiges Prisma Mallia Steatitgruppe</eagle:objectType>

					<eagle:material>weicher Stein</eagle:material>

				</eagle:hasArtifact>

			</eagle:visualRepresentation>

		</eagle:eagleObject>

	</oai:metadata>

</oai:record>

org.apache.cxf.Logger=org.apache.cxf.common.logging.Log4jLogger

log4j.rootLogger=WARN, CONSOLE

log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender

log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout

log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n

log4j.logger.eu.dnetlib=INFO

log4j.logger.eu.dnetlib.data.collector.plugins=DEBUG

log4j.logger.eu.dnetlib.data.collector.plugins.sftp=DEBUG

package prototype.utils;

import com.google.common.base.Function;

public class DotAbbreviations implements Function<String, String> {

	@Override

	public String apply(String s) {

		return s.length() == 1 ? s + "." : s;

	}

}

package prototype.utils;

import com.google.common.base.Function;

import org.apache.commons.lang3.text.WordUtils;

public class Capitalize implements Function<String, String> {

	@Override

	public String apply(String s) {

		return WordUtils.capitalize(s.toLowerCase());

	}

}

package prototype;

import java.nio.charset.StandardCharsets;

import java.text.Normalizer;

import java.util.List;

import java.util.Set;

import com.google.common.base.Joiner;

import com.google.common.base.Splitter;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.google.common.hash.Hashing;

import prototype.utils.Capitalize;

import prototype.utils.DotAbbreviations;

//import eu.dnetlib.pace.clustering.NGramUtils;

//import eu.dnetlib.pace.util.Capitalise;

//import eu.dnetlib.pace.util.DotAbbreviations;

public class Person {

	private static Set<String> particles = null;

	private List<String> name = Lists.newArrayList();

	private List<String> surname = Lists.newArrayList();

	private List<String> fullname = Lists.newArrayList();

	public Person(String s) {

		s = Normalizer.normalize(s, Normalizer.Form.NFD); // was NFD

		s = s.replaceAll("\\(.+\\)", "");

		s = s.replaceAll("\\[.+\\]", "");

		s = s.replaceAll("\\{.+\\}", "");

		s = s.replaceAll("\\s+-\\s+", "-");

		//		s = s.replaceAll("[\\W&&[^,-]]", " ");

		//		System.out.println("class Person: s: " + s);

		//		s = s.replaceAll("[\\p{InCombiningDiacriticalMarks}&&[^,-]]", " ");

		s = s.replaceAll("[\\p{Punct}&&[^-,]]", " ");

		s = s.replaceAll("\\d", " ");

		s = s.replaceAll("\\n", " ");

		s = s.replaceAll("\\.", " ");

		s = s.replaceAll("\\s+", " ");

		if (s.contains(",")) {

			//		System.out.println("class Person: s: " + s);

			String[] arr = s.split(",");

			if (arr.length == 1) {

				fullname = splitTerms(arr[0]);

			} else if (arr.length > 1) {

				surname = splitTerms(arr[0]);

				name = splitTermsFirstName(arr[1]);

				//				System.out.println("class Person: surname: " + surname);

				//				System.out.println("class Person: name: " + name);

				fullname.addAll(surname);

				fullname.addAll(name);

			}

		} else {

			fullname = splitTerms(s);

			int lastInitialPosition = fullname.size();

			boolean hasSurnameInUpperCase = false;

			for (int i = 0; i < fullname.size(); i++) {

				String term = fullname.get(i);

				if (term.length() == 1) {

					lastInitialPosition = i;

				} else if (term.equals(term.toUpperCase())) {

					hasSurnameInUpperCase = true;

				}

			}

			if (lastInitialPosition < fullname.size() - 1) { // Case: Michele G. Artini

				name = fullname.subList(0, lastInitialPosition + 1);

				System.out.println("name: " + name);

				surname = fullname.subList(lastInitialPosition + 1, fullname.size());

			} else if (hasSurnameInUpperCase) { // Case: Michele ARTINI

				for (String term : fullname) {

					if (term.length() > 1 && term.equals(term.toUpperCase())) {

						surname.add(term);

					} else {

						name.add(term);

					}

				}

			} else if (lastInitialPosition == fullname.size()) {

				surname = fullname.subList(lastInitialPosition - 1, fullname.size());

				name = fullname.subList(0, lastInitialPosition - 1);

			}

		}

	}

	private List<String> splitTermsFirstName(String s) {

		List<String> list = Lists.newArrayList();

		for (String part : Splitter.on(" ").omitEmptyStrings().split(s)) {

			if (s.trim().matches("\\p{Lu}{2,3}")) {

				String[] parts = s.trim().split("(?=\\p{Lu})"); // (Unicode UpperCase)

				for (String p : parts) {

					if (p.length() > 0)

						list.add(p);

				}

			} else {

				list.add(part);

			}

		}

		return list;

	}

	private List<String> splitTerms(String s) {

		if (particles == null) {

			//			particles = NGramUtils.loadFromClasspath("/eu/dnetlib/pace/config/name_particles.txt");

		}

		List<String> list = Lists.newArrayList();

		for (String part : Splitter.on(" ").omitEmptyStrings().split(s)) {

			//		if (!particles.contains(part.toLowerCase())) {

			list.add(part);

			//		}

		}

		return list;

	}

	public List<String> getName() {

		return name;

	}

	public List<String> getSurname() {

		return surname;

	}

	public List<String> getFullname() {

		return fullname;

	}

	public String hash() {

		return Hashing.murmur3_128().hashString(getNormalisedFullname(), StandardCharsets.UTF_8).toString();

	}

	public String getNormalisedFullname() {

		return isAccurate() ?

				Joiner.on(" ").join(getSurname()) + ", " + Joiner.on(" ").join(getNameWithAbbreviations()) :

				Joiner.on(" ").join(fullname);

		//				return isAccurate() ?

		//						Joiner.on(" ").join(getCapitalSurname()) + ", " + Joiner.on(" ").join(getNameWithAbbreviations()) :

		//						Joiner.on(" ").join(fullname);

	}

	public List<String> getCapitalSurname() {

		return Lists.newArrayList(Iterables.transform(surname, new Capitalize()));

	}

	public List<String> getNameWithAbbreviations() {

		return Lists.newArrayList(Iterables.transform(name, new DotAbbreviations()));

	}

	public boolean isAccurate() {

		return (name != null && surname != null && !name.isEmpty() && !surname.isEmpty());

	}

}

package prototype;

import java.nio.charset.StandardCharsets;

import java.text.Normalizer;

import java.util.List;

import java.util.Set;

import com.google.common.base.Joiner;

import com.google.common.base.Splitter;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.google.common.hash.Hashing;

import prototype.utils.Capitalize;

import prototype.utils.DotAbbreviations;

//import eu.dnetlib.pace.clustering.NGramUtils;

//import eu.dnetlib.pace.util.Capitalise;

//import eu.dnetlib.pace.util.DotAbbreviations;

public class PersonOrig {

	private static Set<String> particles = null;

	private List<String> name = Lists.newArrayList();

	private List<String> surname = Lists.newArrayList();

	private List<String> fullname = Lists.newArrayList();

	public PersonOrig(String s) {

		s = Normalizer.normalize(s, Normalizer.Form.NFD);

		s = s.replaceAll("\\(.+\\)", "");

		s = s.replaceAll("\\[.+\\]", "");

		s = s.replaceAll("\\{.+\\}", "");

		s = s.replaceAll("\\s+-\\s+", "-");

		s = s.replaceAll("[\\W&&[^,-]]", " ");

		s = s.replaceAll("[\\p{InCombiningDiacriticalMarks}&&[^,-]]", " ");

		s = s.replaceAll("[\\p{Punct}&&[^-,]]", " ");

		s = s.replaceAll("\\d", " ");

		s = s.replaceAll("\\n", " ");

		s = s.replaceAll("\\.", " ");

		s = s.replaceAll("\\s+", " ");

		if (s.contains(",")) {

			String[] arr = s.split(",");

			if (arr.length == 1) {

				fullname = splitTerms(arr[0]);

			} else if (arr.length > 1) {

				surname = splitTerms(arr[0]);

				name = splitTerms(arr[1]);

				fullname.addAll(surname);

				fullname.addAll(name);

			}

		} else {