D-Net

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/cnr-modular-mdstore-service/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "cnr-modular-mdstore-service"}

package eu.dnetlib.enabling.tools.blackboard;

import eu.dnetlib.data.mdstore.modular.action.MDStoreActions;

public class SampleCreateAction implements BlackboardServerAction<MDStoreActions> {

	@Override

	public void execute(BlackboardServerHandler handler, BlackboardJob job) {

		handler.done(job);

	}

}

package eu.dnetlib.enabling.tools.blackboard;

import static org.junit.Assert.assertNotNull;

import static org.mockito.Matchers.anyObject;

import static org.mockito.Matchers.eq;

import static org.mockito.Mockito.mock;

import static org.mockito.Mockito.verify;

import static org.mockito.Mockito.when;

import javax.annotation.Resource;

import org.junit.Before;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.springframework.test.context.ContextConfiguration;

import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import eu.dnetlib.data.mdstore.modular.action.MDStoreActions;

@RunWith(SpringJUnit4ClassRunner.class)

@ContextConfiguration

public class BlackboardServerActionExecutorTest {

	@Resource

	public transient BlackboardServerHandler blackboardHandler;

	@Resource

	public transient BlackboardServerActionExecutor<MDStoreActions> executor;

	@Before

	public void setUp() throws Exception {

	}

	@Test

	public void testExecutor() {

		assertNotNull(executor);

		BlackboardJob job = mock(BlackboardJob.class);

		when(job.getAction()).thenReturn("CREATE");

		executor.execute(job);

		verify(blackboardHandler).done(eq(job));

	}

	@Test

	public void testExecutorUnimplemented() {

		assertNotNull(executor);

		BlackboardJob job = mock(BlackboardJob.class);

		when(job.getAction()).thenReturn("DELETE");

		executor.execute(job);

		verify(blackboardHandler).failed(eq(job), (Throwable) anyObject());

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:p="http://www.springframework.org/schema/p"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">

	<bean id="blackboardHandler" class="eu.dnetlib.test.utils.MockBeanFactory"

		p:clazz="eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler" />

	<bean id="executor"

		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerActionExecutor"

		p:blackboardHandler-ref="blackboardHandler" p:actionType="eu.dnetlib.data.mdstore.modular.action.MDStoreActions"

		p:incomplete="true">

		<property name="actionMap">

			<map>

				<entry key="CREATE">

					<bean class="eu.dnetlib.enabling.tools.blackboard.SampleCreateAction" />

				</entry>

			</map>

		</property>

	</bean>

</beans>

package eu.dnetlib.data.mdstore.modular;

public class MDStoreDescription {

	private String id;

	private String format;

	private String layout;

	private String interpretation;

	private int size;

	private boolean indexed;

	public MDStoreDescription(final String id, final String format, final String layout, final String interpretation, final int size, final boolean indexed) {

		super();

		this.id = id;

		this.format = format;

		this.layout = layout;

		this.interpretation = interpretation;

		this.size = size;

		this.indexed = indexed;

	}

    public MDStoreDescription() {

    }

    public String getId() {

		return id;

	}

	public void setId(final String id) {

		this.id = id;

	}

	public String getFormat() {

		return format;

	}

	public void setFormat(final String format) {

		this.format = format;

	}

	public String getLayout() {

		return layout;

	}

	public void setLayout(final String layout) {

		this.layout = layout;

	}

	public String getInterpretation() {

		return interpretation;

	}

	public void setInterpretation(final String interpretation) {

		this.interpretation = interpretation;

	}

	public int getSize() {

		return size;

	}

	public void setSize(final int size) {

		this.size = size;

	}

	public boolean isIndexed() {

		return indexed;

	}

	public void setIndexed(final boolean indexed) {

		this.indexed = indexed;

	}

}

package eu.dnetlib.data.mdstore.modular;

import com.google.common.base.Function;

import com.google.common.collect.Lists;

import eu.dnetlib.data.mdstore.DocumentNotFoundException;

import eu.dnetlib.data.mdstore.MDStoreService;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.enabling.resultset.IterableResultSetFactory;

import eu.dnetlib.enabling.tools.AbstractBaseService;

import eu.dnetlib.enabling.tools.blackboard.NotificationHandler;

import org.springframework.beans.factory.annotation.Required;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import java.util.List;

import java.util.Map;

public class ModularMDStoreService extends AbstractBaseService implements MDStoreService {

	/**

	 * notification handler.

	 */

	private NotificationHandler notificationHandler;

	private MDStoreFeeder feeder;

	private MDStoreRetriever retriever;

	private IterableResultSetFactory iterableResultSetFactory;

    public List<String> getMDStoreRecords(final String mdId, final int pageSize, final int offset, final Map<String, String> queryParam) throws MDStoreServiceException {

        return retriever.getMDStoreRecords(mdId, pageSize, offset, queryParam);

    }

	@Override

	public W3CEndpointReference deliverMDRecords(final String mdId, final String from, final String until, final String recordFilter)

			throws MDStoreServiceException {

		return retriever.deliver(mdId, from, until, recordFilter);

	}

	@Override

	public W3CEndpointReference bulkDeliverMDRecords(final String format, final String layout, final String interpretation) throws MDStoreServiceException {

		return getIterableResultSetFactory().createIterableResultSet(retriever.deliver(format, layout, interpretation));

	}

	@Override

	public String deliverRecord(final String mdId, final String recordId) throws MDStoreServiceException, DocumentNotFoundException {

		return retriever.deliverRecord(mdId, recordId);

	}

	@Override

	public List<String> getListOfMDStores() throws MDStoreServiceException {

		return (Lists.transform(retriever.getDao().listMDStores(), new Function<MDStoreDescription, String>() {

			@Override

			public String apply(final MDStoreDescription input) {

				return input.getId();

			}

		}));

	}

	@Override

	public List<String> listMDStores(final String format, final String layout, final String interpretation) throws MDStoreServiceException {

		return retriever.getDao().listMDStores(format, layout, interpretation);

	}

	@Override

	public void notify(final String subscriptionId, final String topic, final String isId, final String message) {

		getNotificationHandler().notified(subscriptionId, topic, isId, message);

	}

	@Override

	public boolean storeMDRecordsFromRS(final String mdId, final String rsId, final String storingType) throws MDStoreServiceException {

		throw new MDStoreServiceException("not implemented, use the Blackboard asynchronous equivalent");

	}

	@Override

	public int size(final String mdId) throws MDStoreServiceException {

		return this.getRetriever().getDao().getCachedSize(mdId);

	}

	@Override

	public int size(final String format, final String layout, final String interpretation) throws MDStoreServiceException {

		return this.getRetriever().getDao().getSumOfSizes(format, layout, interpretation);

	}

	public NotificationHandler getNotificationHandler() {

		return notificationHandler;

	}

	@Required

	public void setNotificationHandler(final NotificationHandler notificationHandler) {

		this.notificationHandler = notificationHandler;

	}

	public MDStoreFeeder getFeeder() {

		return feeder;

	}

	public void setFeeder(final MDStoreFeeder feeder) {

		this.feeder = feeder;

	}

	public MDStoreRetriever getRetriever() {

		return retriever;

	}

	public void setRetriever(final MDStoreRetriever retriever) {

		this.retriever = retriever;

	}

	public IterableResultSetFactory getIterableResultSetFactory() {

		return iterableResultSetFactory;

	}

	@Required

	public void setIterableResultSetFactory(final IterableResultSetFactory iterableResultSetFactory) {

		this.iterableResultSetFactory = iterableResultSetFactory;

	}

}

package eu.dnetlib.data.mdstore.modular;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.miscutils.factory.Factory;

public class RecordParserFactory implements Factory<RecordParser> {

	private Class<? extends RecordParser> parserType;

	@Override

	public RecordParser newInstance() {

		try {

			return getParserType().newInstance();

		} catch (Throwable e) {

			throw new RuntimeException(e);

		} 	

	}

	@Required

	public void setParserType(Class<? extends RecordParser> parserType) {

		this.parserType = parserType;

	}

	public Class<? extends RecordParser> getParserType() {

		return parserType;

	}

}

package eu.dnetlib.data.mdstore.modular;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import eu.dnetlib.data.mdstore.DocumentNotFoundException;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.connector.MDStore;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.enabling.resultset.ResultSetFactory;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import java.util.List;

import java.util.Map;

public class MDStoreRetriever {

	/** Logger */

	private static final Log log = LogFactory.getLog(MDStoreRetriever.class);

	private MDStoreDao dao;

	private ResultSetFactory resultSetFactory;

	public W3CEndpointReference deliver(final String mdId, final String from, final String until, final String recordFilter) throws MDStoreServiceException {

		final MDStore mdStore = dao.readMDStore(mdId);

		return getResultSetFactory().createResultSet(mdStore.deliver(from, until, recordFilter));

	}

	public Iterable<String> deliver(final String format, final String layout, final String interpretation) throws MDStoreServiceException {

		try {

			return Iterables.concat(Iterables.transform(dao.listMDStores(format, layout, interpretation), new Function<String, Iterable<String>>() {

				@Override

				public Iterable<String> apply(final String mdId) {

					log.debug("bulk deliver of mdId: " + mdId);

					try {

						return dao.readMDStore(mdId).iterate();

					} catch (MDStoreServiceException e) {

						throw new RuntimeException(e);

					}

				}

			}));

		} catch (RuntimeException e) {

			throw new MDStoreServiceException(e);

		}

	}

	public String deliverRecord(final String mdId, final String recordId) throws MDStoreServiceException, DocumentNotFoundException {

		return dao.getMDStore(mdId).getRecord(recordId);

	}

	public MDStoreDao getDao() {

		return dao;

	}

	@Required

	public void setDao(final MDStoreDao dao) {

		this.dao = dao;

	}

    public ResultSetFactory getResultSetFactory() {

        return resultSetFactory;

    }

	@Required

	public void setResultSetFactory(final ResultSetFactory resultSetFactory) {

		this.resultSetFactory = resultSetFactory;

	}

    public List<String> getMDStoreRecords(final String mdId, int pageSize, final int offset, final Map<String, String> queryParam) throws MDStoreServiceException {

        MDStore mdStore = dao.getMDStore(mdId);

        return mdStore.deliver(mdId, pageSize, offset, queryParam);

    }

}

package eu.dnetlib.data.mdstore.modular;

import java.io.StringReader;

import java.util.HashMap;

import java.util.Map;

import javax.xml.xpath.XPath;

import javax.xml.xpath.XPathFactory;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.xml.sax.InputSource;

import static eu.dnetlib.data.mdstore.modular.MDStoreConstants.*;

/**

 * Terrible implementation of a record parser.

 * 

 * @author marko

 *

 */

public class SimpleRecordParser implements RecordParser {

	static final Log log = LogFactory.getLog(SimpleRecordParser.class); // NOPMD by marko on 11/24/08 5:02 PM

	private long ts;

	@Override

	public Map<String, String> parseRecord(String record) {

		Map<String, String> props = new HashMap<String, String>();

		props.put(TIMESTAMP, String.valueOf(getTimestamp()));

		try {

//			DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();

			XPath xpath = XPathFactory.newInstance().newXPath();

//			Document doc = builder.parse(new InputSource(new StringReader(record)));

			InputSource doc = new InputSource(new StringReader(record));

			props.put(ID, xpath.evaluate("//*[local-name()='objIdentifier']", doc));

			props.put("originalId", xpath.evaluate("//*[local-name()='efgEntity']/*/*[local-name()='identifier']", doc));

//			String date = xpath.evaluate("//*[local-name()='dateOfCollection'][1]", doc);

//			props.put("date", new Date(date).getTime());	

		} catch (Exception e) {

			log.warn("got exception while parsing document", e);

			log.warn("record is:");

			log.warn(record);

			log.warn("------------");

		}

		return props;

	}

	@Override

	public void setTimestamp(final long ts) {

		this.ts = ts;

		log.debug("RecordParser date set to "+ts);

	}

	@Override

	public long getTimestamp() {

		return ts;

	}

}

package eu.dnetlib.data.mdstore.modular;

import java.io.ByteArrayInputStream;

import java.util.HashMap;

import java.util.Map;

import java.util.Stack;

import javax.xml.stream.XMLInputFactory;

import javax.xml.stream.XMLStreamConstants;

import javax.xml.stream.XMLStreamException;

import javax.xml.stream.XMLStreamReader;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import static eu.dnetlib.data.mdstore.modular.MDStoreConstants.*;

/**

 * This method outperforms SimpleRecordParser by a vast amount, especially since we are just getting stuff in the

 * header.

 * 

 * @author marko

 * 

 */

public class StreamingRecordParser implements RecordParser {

	private static final Log log = LogFactory.getLog(StreamingRecordParser.class);

	private long ts;

	@Override

	public Map<String, String> parseRecord(String record) {

		try {

			XMLInputFactory factory = XMLInputFactory.newInstance();

			XMLStreamReader parser = factory.createXMLStreamReader(new ByteArrayInputStream(record.getBytes()));

			HashMap<String, String> res = new HashMap<String, String>();

			res.put(TIMESTAMP, String.valueOf(getTimestamp()));

			Stack<String> elementStack = new Stack<String>();

			elementStack.push("/");

			while (parser.hasNext()) {

				int event = parser.next();

				if (event == XMLStreamConstants.END_ELEMENT) {

					elementStack.pop();

				} else if (event == XMLStreamConstants.START_ELEMENT) {

					final String localName = parser.getLocalName();

					elementStack.push(localName);

					if (OBJIDENTIFIER.equals(localName)) {

						parser.next();

						res.put(ID, parser.getText().trim());

					} else if ("identifier".equals(localName) && "efgEntity".equals(grandParent(elementStack))) {

						if (!res.containsKey("originalId")) {

							parser.next();

//							log.info("ZZZZZZ OK: found identifier at right depth " + elementStack);

							res.put("originalId", parser.getText().trim());

						}

					}

					else if ("identifier".equals(localName)) {

//						log.info("ZZZZZZ: found identifier not at right depth " + elementStack + " grand parent " + grandParent(elementStack));

					}

					if (res.containsKey(ID) && res.containsKey("originalId"))

						return res;

				}

			}

			return res;

		} catch (XMLStreamException e) {

			throw new IllegalStateException(e);

		}

	}

	private String grandParent(Stack<String> elementStack) {

		if (elementStack.size() <= 3)

			return "";

		return elementStack.get(elementStack.size() - 3);

	}

	@Override

	public void setTimestamp(final long ts) {

		this.ts = ts;

		log.debug("RecordParser date set to "+ts);

	}

	@Override

	public long getTimestamp() {

		return ts;

	}

}

package eu.dnetlib.data.mdstore.modular;

import com.google.common.collect.Maps;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.action.DoneCallback;

import eu.dnetlib.data.mdstore.modular.action.FailedCallback;

import eu.dnetlib.data.mdstore.modular.connector.MDStore;

import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;

import eu.dnetlib.miscutils.datetime.DateUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import java.util.List;

import java.util.Map;

public class MDStoreFeeder {

	private static final Log log = LogFactory.getLog(MDStoreFeeder.class);

	private MDStoreDao dao;

	private ResultSetClientFactory resultSetClientFactory;

	private UniqueServiceLocator serviceLocator;

	private boolean syncFeed = true;

	public void feed(final String mdId,

                     final String rsEpr,

                     final String storingType,

                     final boolean sync,

                     final List<MDFormatDescription> mdformats,

                     final DoneCallback doneCallback,

                     final FailedCallback failCallback) throws MDStoreServiceException {

		log.info("Start feeding mdstore " + mdId);

		log.debug("Start feeding mdstore " + mdId + " with epr " + rsEpr);

		String transactionId = null;

		try {

			final boolean refresh = "REFRESH".equals(storingType);

			final MDStore mdstore = dao.startTransaction(mdId, refresh);

			transactionId = mdstore.getId();

			final Iterable<String> records = resultSetClientFactory.getClient(rsEpr);

			if (refresh) {

				mdstore.truncate();

			}

            int writeOps;

            if (mdformats == null) {

                writeOps = mdstore.feed(records, refresh);

            } else {

                writeOps = mdstore.feed(records, refresh, mdformats);

            }

			dao.commit(mdstore.getId(), mdId);

			int size = dao.refreshSize(mdId);

			touch(mdId, size);

			log.info("Finished feeding mdstore " + mdId + " - new size: " + size);

			doneCallback.call(buildParams(size, writeOps));

		} catch (Throwable e) {

			if (transactionId != null) {

				dao.invalidTransaction(transactionId, mdId);

			}

			log.error("Error feeding mdstore: " + mdId);

			failCallback.call(e);

		}

	}

	private Map<String, String> buildParams(final int size, final int storeCount) {

		Map<String, String> params = Maps.newHashMap();

		params.put("mdstoreSize", String.valueOf(size));

		params.put("writeOps", String.valueOf(storeCount));

		return params;

	}

	/**

	 * Sets the last modified date in the profile.

	 *

	 * @param mdId

	 */

	public void touch(final String mdId, final int size) {

		try {

			final String now = DateUtils.now_ISO8601();

			final String mdstoreXUpdate = "for $x in //RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + mdId + "']"

					+ "return update value $x//LAST_STORAGE_DATE with '" + now + "'";

			serviceLocator.getService(ISRegistryService.class).executeXUpdate(mdstoreXUpdate);

			touchSize(mdId, size);

		} catch (final Exception e) {

			throw new IllegalStateException(e);

		}

	}

	public void touchSize(final String mdId, final int size) {

		try {

			final String mdstoreNumberXUpdate = "for $x in //RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + mdId + "']"

					+ "return update value $x//NUMBER_OF_RECORDS with '" + size + "'";

			serviceLocator.getService(ISRegistryService.class).executeXUpdate(mdstoreNumberXUpdate);

		} catch (final Exception e) {

			throw new IllegalStateException(e);

		}

	}

	public MDStoreDao getDao() {

		return dao;

	}

	@Required

	public void setDao(final MDStoreDao dao) {

		this.dao = dao;

	}

	public ResultSetClientFactory getResultSetClientFactory() {

		return resultSetClientFactory;

	}

	@Required

	public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {

		this.resultSetClientFactory = resultSetClientFactory;

	}

	public boolean isSyncFeed() {

		return syncFeed;

	}

	public void setSyncFeed(final boolean syncFeed) {

		this.syncFeed = syncFeed;

	}

	public UniqueServiceLocator getServiceLocator() {

		return serviceLocator;

	}

	@Required

	public void setServiceLocator(final UniqueServiceLocator serviceLocator) {

		this.serviceLocator = serviceLocator;

	}

}

package eu.dnetlib.data.mdstore.modular;

import java.io.StringReader;

import java.io.StringWriter;

import java.util.List;

import javax.xml.namespace.QName;

import javax.xml.stream.XMLEventFactory;

import javax.xml.stream.XMLEventReader;

import javax.xml.stream.XMLEventWriter;

import javax.xml.stream.XMLInputFactory;

import javax.xml.stream.XMLOutputFactory;

import javax.xml.stream.XMLStreamException;

import javax.xml.stream.events.Namespace;

import javax.xml.stream.events.StartElement;

import javax.xml.stream.events.XMLEvent;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.google.common.base.Function;

import com.google.common.collect.Lists;

import eu.dnetlib.miscutils.collections.Pair;

import eu.dnetlib.miscutils.factory.Factory;

public class BulkRecordMapperFactory implements Factory<Function<String, Pair<String, String>>> {

	private static final Log log = LogFactory.getLog(BulkRecordMapperFactory.class); // NOPMD by marko on 11/24/08 5:02 PM

	protected static final String MD_RECORD = "mdRecord";

	protected static final String MD_ID = "mdId";

	protected static final String RECORD = "record";

	protected ThreadLocal<XMLInputFactory> inputFactory = new ThreadLocal<XMLInputFactory>() {

		@Override

		protected XMLInputFactory initialValue() {

			return XMLInputFactory.newInstance();

		}

	};

	protected ThreadLocal<XMLOutputFactory> outputFactory = new ThreadLocal<XMLOutputFactory>() {

		@Override

		protected XMLOutputFactory initialValue() {

			return XMLOutputFactory.newInstance();

		}

	};

	protected ThreadLocal<XMLEventFactory> eventFactory = new ThreadLocal<XMLEventFactory>() {

		@Override

		protected XMLEventFactory initialValue() {

			return XMLEventFactory.newInstance();

		}

	};

	@Override

	public Function<String, Pair<String, String>> newInstance() {

		return new Function<String, Pair<String, String>>() {

			private String mdId = null;

			private String record = null;

			@Override

			public Pair<String, String> apply(String embeddedRecord) {

				try {

					final XMLEventReader parser = inputFactory.get().createXMLEventReader(new StringReader(embeddedRecord));

					while (parser.hasNext()) {

						final XMLEvent event = parser.nextEvent();

						if (event != null && event.isStartElement()) {

							final String localName = event.asStartElement().getName().getLocalPart();

							if (MD_RECORD.equals(localName)) {

								mdId = event.asStartElement().getAttributeByName(new QName(MD_ID)).getValue();

							} else if (RECORD.equals(localName)) {

								record = getRecord(embeddedRecord, parser);

							} 

						}

					}

				} catch (final XMLStreamException e) {

					log.error("error parsing record: " + embeddedRecord);

				}				

				return new Pair<String, String>(mdId, record);

			}

		};

	}

	/**

	 * Copy the /indexRecord/result element and children, preserving namespace declarations etc.

	 * 

	 * @param indexDocument

	 * @param results

	 * @param parser

	 * @throws XMLStreamException

	 */

	protected String getRecord(final String record, final XMLEventReader parser) throws XMLStreamException {

		StringWriter results = new StringWriter();

		final XMLEventWriter writer = outputFactory.get().createXMLEventWriter(results);

		// TODO: newRecord should copy all the namespace prefixes setup in parents

		// fortunately the only parent of the result element is the 'indexrecord', so it should be easy to get

		// the namespaces declared on the root element (and fast)

		final List<Namespace> namespaces = Lists.newArrayList(

				eventFactory.get().createNamespace("dri", "http://www.driver-repository.eu/namespace/dri"),

				eventFactory.get().createNamespace("dr", "http://www.driver-repository.eu/namespace/dr"),

				eventFactory.get().createNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance"),

				eventFactory.get().createNamespace("dc", "http://purl.org/dc/elements/1.1/"));

		StartElement newRecord = eventFactory.get().createStartElement("", null, RECORD, null, namespaces.iterator());

		// new root record

		writer.add(newRecord);

		// copy the rest as it is

		while (parser.hasNext()) {

			final XMLEvent resultEvent = parser.nextEvent();

			// TODO: replace with depth tracking instead of close tag tracking. 

			if (resultEvent.isEndElement() && resultEvent.asEndElement().getName().getLocalPart().equals(RECORD)) {

				writer.add(resultEvent);

				break;

			}

			writer.add(resultEvent);

		}

		writer.close();

		return results.toString();

	}

}

package eu.dnetlib.data.mdstore.modular;

public class MDStoreConstants {

	public static final String ID =  "id";

	public static final String OBJIDENTIFIER =  "objIdentifier";

	public static final String TIMESTAMP = "timestamp";

	public static final String ORIGINALID = "originalId";

	public static final String BODY = "body";

}

package eu.dnetlib.data.mdstore.modular.connector;

public class MDStoreResults {

	private Iterable<String> records;

	private int size;

	public MDStoreResults(Iterable<String> records, int size) {

		super();

		this.records = records;

		this.size = size;

	}

	public int getSize() {

		return size;

	}

	public void setSize(int size) {

		this.size = size;

	}

	public Iterable<String> getRecords() {

		return records;

	}

	public void setRecords(Iterable<String> records) {

		this.records = records;

	}

}

package eu.dnetlib.data.mdstore.modular.connector;

import java.util.Date;

public class MDStoreTransactionInfo {

	private String id;

	private Boolean refresh;

	private Date date;

	private long size;

	/**

	 * @return the date

	 */

	public Date getDate() {

		return date;

	}

	/**

	 * @param date

	 *            the date to set

	 */

	public void setDate(final Date date) {

		this.date = date;

	}

	/**

	 * @return the id

	 */

	public String getId() {

		return id;

	}

	/**

	 * @param id

	 *            the id to set

	 */

	public void setId(final String id) {

		this.id = id;

	}

	/**

	 * @return the refresh

	 */

	public Boolean getRefresh() {

		return refresh;

	}

	/**

	 * @param refresh

	 *            the refresh to set

	 */

	public void setRefresh(final Boolean refresh) {

		this.refresh = refresh;

	}

	/**

	 * @return the size

	 */

	public long getSize() {

		return size;

	}

	/**

	 * @param size

	 *            the size to set

	 */

	public void setSize(final long size) {

		this.size = size;

	}

}

package eu.dnetlib.data.mdstore.modular.connector;

import java.util.List;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.MDStoreDescription;

public interface MDStoreDao {

	MDStore getMDStore(final String mdId) throws MDStoreServiceException;

	MDStore readMDStore(final String mdId) throws MDStoreServiceException;

	MDStore startTransaction(final String mdId, final boolean refresh) throws MDStoreServiceException;

	void commit(final String transactionId, final String mdId) throws MDStoreServiceException;

	List<MDStoreDescription> listMDStores() throws MDStoreServiceException;

	List<String> listMDStores(final String format, final String layout, final String interpretation) throws MDStoreServiceException;

	void createMDStore(final String mdId, final String format, final String interpretation, final String layout) throws MDStoreServiceException;

	void deleteMDStore(final String id) throws MDStoreServiceException;

	int getCachedSize(final String id) throws MDStoreServiceException;

	void refreshSizes() throws MDStoreServiceException;

	int refreshSize(final String id) throws MDStoreServiceException;

	int getSumOfSizes(final String format, final String layout, final String interpretation) throws MDStoreServiceException;

	void startGarbage() throws MDStoreServiceException;

	void invalidTransaction(final String transactionId, final String mdId) throws MDStoreServiceException;

	MDStoreDBStatus getDBStatus();

}

package eu.dnetlib.data.mdstore.modular.connector;

import java.util.ArrayList;

import java.util.List;

public class MDStoreManagerInfo {

	private String mdId;

	private String currentId;

	private List<MDStoreExpiredInfo> stillUsed;

	private List<MDStoreTransactionInfo> transactions;

	public String getMdId() {

		return mdId;

	}

	public void setMdId(final String mdId) {

		this.mdId = mdId;

	}

	public String getCurrentId() {

		return currentId;

	}

	public void setCurrentId(final String currentId) {

		this.currentId = currentId;

	}

	public List<MDStoreExpiredInfo> getStillUsed() {

		return stillUsed;

	}

	public void setStillUsed(final List<MDStoreExpiredInfo> stillUsed) {

		this.stillUsed = stillUsed;

	}

	public List<MDStoreTransactionInfo> getTransactions() {

		return transactions;

	}

	public void setTransactions(final List<MDStoreTransactionInfo> transactions) {

		this.transactions = transactions;

	}

	public void addExpiredItem(final MDStoreExpiredInfo info) {

		if (stillUsed == null) {

			stillUsed = new ArrayList<MDStoreExpiredInfo>();

		}

		stillUsed.add(info);

	}

	public void addTransactionInfo(final MDStoreTransactionInfo info) {

		if (transactions == null) {

			transactions = new ArrayList<MDStoreTransactionInfo>();

		}

		transactions.add(info);

	}

}

package eu.dnetlib.data.mdstore.modular.connector;

public class MDStoreDBStatus {

	private int handledDatastructures;

	private int usedDiskSpace;

	private String date;

	public MDStoreDBStatus() {}

	public MDStoreDBStatus(final int handledDatastructures, final int usedDiskSpace, final String date) {

		this.handledDatastructures = handledDatastructures;

		this.usedDiskSpace = usedDiskSpace;

		this.date = date;

	}

	public int getHandledDatastructures() {

		return handledDatastructures;

	}

	public void setHandledDatastructures(final int handledDatastructures) {

		this.handledDatastructures = handledDatastructures;

	}

	public int getUsedDiskSpace() {

		return usedDiskSpace;

	}

	public void setUsedDiskSpace(final int usedDiskSpace) {

		this.usedDiskSpace = usedDiskSpace;

	}

	public String getDate() {

		return date;

	}

	public void setDate(final String date) {

		this.date = date;

	}

	@Override

	public String toString() {

		return "MDStoreDBStatus{" +

				"handledDatastructures=" + handledDatastructures +

				", usedDiskSpace=" + usedDiskSpace +

				", date='" + date + '\'' +

				'}';

	}

}

package eu.dnetlib.data.mdstore.modular.connector;

import java.util.Date;

public class MDStoreExpiredInfo {

	private String id;

	private Date lastRead;

	/**

	 * @return the lastRead

	 */

	public Date getLastRead() {

		return lastRead;

	}

	/**

	 * @param lastRead

	 *            the lastRead to set

	 */

	public void setLastRead(final Date lastRead) {

		this.lastRead = lastRead;

	}

	/**

	 * @return the id

	 */

	public String getId() {

		return id;

	}

	/**

	 * @param id

	 *            the id to set

	 */

	public void setId(final String id) {

		this.id = id;

	}

}

package eu.dnetlib.data.mdstore.modular.connector;

import java.util.List;

import java.util.Map;

import eu.dnetlib.data.mdstore.DocumentNotFoundException;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.data.mdstore.modular.MDFormatDescription;

import eu.dnetlib.enabling.resultset.ResultSetListener;

public interface MDStore {

	String getId();

	String getFormat();

	String getInterpretation();

	String getLayout();

	void truncate();

	int feed(Iterable<String> records, boolean incremental);

    int feed(Iterable<String> records, boolean incremental, List<MDFormatDescription> mdformats);

	ResultSetListener deliver(String from, String until, String recordFilter) throws MDStoreServiceException;

	ResultSetListener deliverIds(String from, String until, String recordFilter) throws MDStoreServiceException;

	Iterable<String> iterate();

	int getSize();

	void deleteRecord(String recordId);

	String getRecord(String recordId) throws DocumentNotFoundException;

    List<String> deliver(String mdId, int pageSize, int offset, Map<String, String> queryParam);

}

package eu.dnetlib.data.mdstore.modular.connector;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

public interface MDStoreTransactionManager {

	/**

	 * Verify consistency.

	 *

	 * @throws MDStoreServiceException

	 *             the MD store service exception

	 */

	public abstract void verifyConsistency() throws MDStoreServiceException;

	/**

	 * Creates the md store.

	 *

	 * @param mdId

	 *            the md id

	 * @throws MDStoreServiceException

	 *             the MD store service exception

	 */

	public abstract void createMDStore(String mdId) throws MDStoreServiceException;

	/**

	 * Drop md store.

	 *

	 * @param mdId

	 *            the md id

	 * @throws MDStoreServiceException

	 *             the MD store service exception

	 */

	public abstract void dropMDStore(String mdId) throws MDStoreServiceException;

	/**

	 * Gets the MD store collection.

	 *

	 * @param mdId

	 *            the md id

	 * @return the MD store collection

	 * @throws MDStoreServiceException

	 *             the MD store service exception

	 */

	public abstract String getMDStoreCollection(String mdId) throws MDStoreServiceException;

	/**

	 * Start a new transaction for writing in the mdstore this will create a temporary mdstore in which save the content. and after finished

	 * switch it to the existing one

	 *

	 * @param mdId

	 * @param refresh

	 * @throws MDStoreServiceException

	 *             if the mdStore Id doesn't exists in the metadata

	 */

	public abstract String startTransaction(String mdId, boolean refresh) throws MDStoreServiceException;

	/**

	 * Commit the transaction.

	 *

	 * @param transactionId

	 *            the transaction id

	 * @param mdstoreId

	 *            the mdstore id

	 * @param currentMDStore

	 *            the current md store

	 * @return true, if successful

	 * @throws MDStoreServiceException

	 *             the MD store service exception

	 */

	public abstract boolean commit(String transactionId, String mdstoreId, MDStore currentMDStore) throws MDStoreServiceException;

	/**

	 * Book a a current mdstore, so the transaction manager can't delete during a possible commit, until the resultset associate has not

	 * expired

	 *

	 * @param mdStoreId

	 * @return

	 * @throws MDStoreServiceException

	 */

	public abstract String readMdStore(String mdStoreId) throws MDStoreServiceException;

	/**

	 * Return a JSON about the transaction assigned to a particular mdstore

	 *

	 * @param mdStoreId

	 * @return

	 * @throws MDStoreServiceException

	 */

	public abstract MDStoreManagerInfo getInfoForCurrentMdStore(String mdStoreId) throws MDStoreServiceException;

	/**

	 * Manually drop an old collection assigned to a particular mdStore

	 *

	 * @param mdId

	 *            : the id of the mdStore

	 * @param idToDrop

	 *            : The id of the collection to be dropped

	 * @return

	 * @throws MDStoreServiceException

	 */

	public abstract Boolean dropUsed(String mdId, String idToDrop) throws MDStoreServiceException;

	/**

	 * Manually drop an old collection assigned to a particular mdStore

	 *

	 * @param mdId

	 *            : the id of the mdStore

	 * @param idToDrop

	 *            : The id of the collection to be dropped

	 * @return

	 * @throws MDStoreServiceException

	 */

	public abstract Boolean dropTransaction(String mdId, String idToDrop) throws MDStoreServiceException;

	/**

	 * Start the garbage collection of the old mdstore not used

	 *

	 * @throws MDStoreServiceException

	 */

	public abstract void garbage() throws MDStoreServiceException;

}

package eu.dnetlib.data.mdstore.modular;

import java.util.Map;

import eu.dnetlib.miscutils.datetime.DateUtils;

/**

 * Parses a mdrecord and extracts the minimum information (like id, date etc) which is necessary for the mdstoring

 * process.

 * 

 * @author marko

 * 

 */

public interface RecordParser {

	Map<String, String> parseRecord(String record);

	void setTimestamp(long ts);

	default long getTimestamp() {

		return DateUtils.now();

	}

}

package eu.dnetlib.data.mdstore.modular;

/**

 * Created by sandro on 11/29/16.

 */

public class MDFormatDescription {

    private String name;

    private String xpath;

    public MDFormatDescription() {

    }

    public MDFormatDescription(String name, String xpath) {

        this.name = name;

        this.xpath = xpath;

    }

    public String getName() {

        return name;

    }

    public void setName(String name) {

        this.name = name;

    }

    public String getXpath() {

        return xpath;

    }

    public void setXpath(String xpath) {

        this.xpath = xpath;

    }

}