D-Net

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>cnr-data-information-oai-publisher</artifactId>

	<packaging>jar</packaging>

	<version>8.0.0</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/cnr-data-information-oai-publisher/tags/cnr-data-information-oai-publisher-8.0.0</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-data-information-oai-publisher-common</artifactId>

			<version>[6.0.0,7.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-oai-store-service</artifactId>

			<version>[8.0.0,9.0.0)</version>

		</dependency>

		<dependency>

			<groupId>org.springframework</groupId>

			<artifactId>spring-aop</artifactId>

			<version>${spring.version}</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

	</dependencies>

</project>

{"type_source": "SVN", "goal": "package -U source:jar",

"url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/cnr-data-information-oai-publisher/trunk/",

"deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it",

"deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "cnr-data-information-oai-publisher"}

package eu.dnetlib.data.information.oai.publisher.core;

import java.util.Iterator;

import java.util.List;

import javax.annotation.Resource;

import com.google.common.collect.Lists;

import eu.dnetlib.data.information.oai.publisher.*;

import eu.dnetlib.data.information.oai.publisher.conf.OAIConfigurationReader;

import eu.dnetlib.data.information.oai.publisher.info.ListDocumentsInfo;

import eu.dnetlib.data.information.oai.publisher.info.MDFInfo;

import eu.dnetlib.data.information.oai.publisher.info.RecordInfo;

import eu.dnetlib.data.information.oai.publisher.info.ResumptionTokenImpl;

import eu.dnetlib.data.oai.store.Cursor;

import eu.dnetlib.data.oai.store.PublisherStore;

import eu.dnetlib.data.oai.store.PublisherStoreDAO;

import eu.dnetlib.miscutils.functional.UnaryFunction;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

public class DNetOAICore extends AbstractOAICore {

	private static final Log log = LogFactory.getLog(DNetOAICore.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Resource(name = "mongodbPublisherStoreDao")

	private PublisherStoreDAO<PublisherStore<Cursor>, Cursor> publisherStoreDAO;

	private String defaultDate = "2008-01-01T12:00:00Z";

	@Override

	protected RecordInfo getRecordById(final MDFInfo mdf, final String id) throws OaiPublisherException {

		PublisherStore<Cursor> store = this.publisherStoreDAO.getStoreFor(mdf.getPrefix(), getCurrentDBName());

		if (store == null)

			throw new OaiPublisherRuntimeException("Missing store for metadata prefix " + mdf.getPrefix() + ". Please check OAI publisher configuration.");

		RecordInfo record = null;

		if (StringUtils.isBlank(mdf.getTransformationRuleID())) {

			record = store.getRecord(id);

		} else {

			UnaryFunction<String, String> function = getLookupClient().getUnaryFunctionFromTDSRule(mdf.getTransformationRuleID());

			record = store.getRecord(id, function);

		}

		if (record != null) {

			record.setPrefix(mdf.getPrefix());

		}

		return record;

	}

	/**

	 * 

	 * {@inheritDoc}

	 * 

	 * @see eu.dnetlib.data.information.oai.publisher.core.AbstractOAICore#getDocuments(boolean, java.lang.String, java.lang.String,

	 *      java.lang.String, java.lang.String)

	 */

	@Override

	protected ListDocumentsInfo getDocuments(final boolean onlyIdentifiers, final String set, final String metadataPrefix, final String from, final String until)

			throws OaiPublisherException {

		MDFInfo mdf = obtainMDFInfo(metadataPrefix);

		boolean hasDateRange = StringUtils.isNotBlank(from) || StringUtils.isNotBlank(until);

		String query = this.generateQuery(mdf, set, from, until, hasDateRange);

		int total = this.countTotal(hasDateRange, query, set, mdf);

		log.debug("Total number of records: " + total);

		Cursor results = this.getCursor(query, onlyIdentifiers, mdf);

		ListDocumentsInfo res = this.prepareListDocumentsInfo(results, mdf, query, set, 0, total, hasDateRange);

		log.debug("Delivering " + res.getDocs().size() + " in a page");

		return res;

	}

	@Override

	protected ListDocumentsInfo getDocuments(final boolean onlyIdentifiers, final String resumptionToken) throws OaiPublisherException {

		ResumptionTokenImpl resToken = new ResumptionTokenImpl();

		resToken.deserialize(resumptionToken);

		log.debug(resToken.toString());

		MDFInfo mdf = obtainMDFInfo(resToken.getMetadataPrefix());

		String lastID = resToken.getLastObjIdentifier();

		String query = resToken.getQuery();

		String newQuery = "";

		if (StringUtils.isNotBlank(query)) {

			newQuery = query + " AND ";

		}

		newQuery += " _id > \"" + lastID + "\"";

		log.debug("NEW QUERY BECAUSE of resumptionToken: " + newQuery);

		int total = this.countTotal(resToken.hasDateRange(), query, resToken.getRequestedSet(), mdf);

		Cursor results = this.getCursor(newQuery, onlyIdentifiers, mdf);

		int oldCount = resToken.getnMaxElements();

		// if the number of records changed, then for sure we can invalidate the resumption token, unless we have a new total of -1 (date

		// range queries can't be counted for performance reasons)

		if ((total != -1) && (oldCount != total)) throw new BadResumptionTokenException(resumptionToken);

		ListDocumentsInfo res = this.prepareListDocumentsInfo(results, mdf, query, resToken.getRequestedSet(), resToken.getnRead(), resToken.getnMaxElements(),

				resToken.hasDateRange());

		res.setCursor(resToken.getnRead());

		return res;

	}

	protected ListDocumentsInfo prepareListDocumentsInfo(final Cursor results,

			final MDFInfo mdf,

			final String query,

			final String requestedSet,

			final int read,

			final int totalNumber,

			final boolean hasDateRange) throws OaiPublisherException {

		ListDocumentsInfo documentList = new ListDocumentsInfo();

		documentList.setnMaxElements(totalNumber);

		documentList.setMetadataPrefix(mdf.getPrefix());

		documentList.setCursor(0);

		if (documentList.getnMaxElements() == 0) throw new NoRecordsMatchException(OAIError.noRecordsMatch.getMessage());

		List<RecordInfo> theRecords = this.generateOAIRecords(mdf, requestedSet, results);

		documentList.setDocs(theRecords);

		if ((theRecords == null) || theRecords.isEmpty()) throw new NoRecordsMatchException("noRecordsMatch: 'documents' is null or empty");

		if ((documentList.getnMaxElements() > (read + theRecords.size())) || (documentList.getnMaxElements() == -1)) {

			String lastID = theRecords.get(theRecords.size() - 1).getInternalId();

			ResumptionTokenImpl nextToken = new ResumptionTokenImpl();

			nextToken.setDateRange(hasDateRange);

			nextToken.setLastObjIdentifier(lastID);

			nextToken.setMetadataPrefix(mdf.getPrefix());

			nextToken.setnMaxElements(totalNumber);

			nextToken.setnRead(read + theRecords.size());

			nextToken.setQuery(query);

			nextToken.setRequestedSet(requestedSet);

			documentList.setResumptionToken(nextToken);

		}

		return documentList;

	}

	protected Cursor getCursor(final String query, final boolean onlyIdentifiers, final MDFInfo mdfInfo) {

		PublisherStore<Cursor> store = this.publisherStoreDAO.getStore(mdfInfo.getSourceFormatName(), mdfInfo.getSourceFormatInterpretation(),

				mdfInfo.getSourceFormatLayout(), getCurrentDBName());

		if (store == null)

			throw new OaiPublisherRuntimeException("Missing store for metadata prefix " + mdfInfo.getPrefix() + ". Please check OAI publisher configuration.");

		Cursor results = null;

		if (StringUtils.isBlank(mdfInfo.getTransformationRuleID())) {

			results = store.getRecords(query, !onlyIdentifiers, pageSize);

		} else {

			UnaryFunction<String, String> function = getLookupClient().getUnaryFunctionFromTDSRule(mdfInfo.getTransformationRuleID());

			results = store.getRecords(query, function, !onlyIdentifiers, pageSize);

		}

		return results;

	}

	/**

	 * Generates the List of RecordInfo to be delivered.

	 *

	 * @param mdf

	 *            MDFInfo, the requested metadata format information.

	 * @param requestedSet

	 *            set specified in the request. It is blank if no set was requested.

	 * @param cursor

	 *            Cursor instance to use to get the records.

	 * @return List of RecordInfo instances

	 */

	protected List<RecordInfo> generateOAIRecords(final MDFInfo mdf, final String requestedSet, final Cursor cursor) {

		final List<RecordInfo> documents = Lists.newArrayList();

		Iterator<RecordInfo> cursorIterator = cursor.iterator();

		while (cursorIterator.hasNext()) {

			RecordInfo current = cursorIterator.next();

			current.addSetspec(requestedSet);

			current.setPrefix(mdf.getPrefix());

			documents.add(current);

		}

		return documents;

	}

	protected String generateQuery(final MDFInfo mdf, final String set, final String from, final String until, final boolean hasDateRange) {

		String datestampIndexName = OAIConfigurationReader.DATESTAMP_FIELD;

		String query = mdf.getBaseQuery();

		if (!StringUtils.isBlank(set)) {

			if (!StringUtils.isBlank(query)) {

				query += " AND ";

			}

			query += getSetCollection().getSetQuery(set, getCurrentDBName());

		}

		if (hasDateRange) {

			if (!StringUtils.isBlank(query)) {

				query += " AND ";

			}

			if ((from != null) && (until != null)) {

				query += datestampIndexName + " >= " + from + " AND " + datestampIndexName + " <= " + until;

			} else if (from != null) {

				query += datestampIndexName + " >= " + from;

			} else if (until != null) {

				query += datestampIndexName + " <= " + until;

			}

		}

		log.info("QUERY GENERATED: \n" + query);

		return query;

	}

	private int countTotal(final boolean hasDateRange, final String query, final String set, final MDFInfo mdFormat) {

		int total = 0;

		if (hasDateRange) {

			// Counting in the store by date ranges is too expensive and delays to much the response

			total = -1;

		} else {

			String theSet = set;

			if (StringUtils.isBlank(set)) {

				theSet = "ALL";

			}

			log.debug("SET::: " + theSet);

			total = getSetCollection().count(theSet, mdFormat.getPrefix(), getCurrentDBName());

		}

		return total;

	}

	public String getDefaultDate() {

		return defaultDate;

	}

	public void setDefaultDate(final String defaultDate) {

		this.defaultDate = defaultDate;

	}

	public PublisherStoreDAO<PublisherStore<Cursor>, Cursor> getPublisherStoreDAO() {

		return publisherStoreDAO;

	}

	public void setPublisherStoreDAO(final PublisherStoreDAO<PublisherStore<Cursor>, Cursor> publisherStoreDAO) {

		this.publisherStoreDAO = publisherStoreDAO;

	}

	public int getPageSize() {

		return pageSize;

	}

	public void setPageSize(final int pageSize) {

		this.pageSize = pageSize;

	}

}

package eu.dnetlib.data.information.oai.publisher.core;

import org.aopalliance.intercept.MethodInvocation;

import org.apache.commons.logging.Log;

import org.springframework.aop.interceptor.CustomizableTraceInterceptor;

/**

 * This class intercepts calls to the OAICore to measure its execution time. It uses Spring AOP.

 * 

 * @author alessia

 * 

 */

public class CoreInterceptor extends CustomizableTraceInterceptor {

	private static final long serialVersionUID = -9063818317778608736L;

	@Override

	protected void writeToLog(final Log logger, final String message, final Throwable ex) {

		if (ex != null) {

			logger.error(message, ex);

		} else {

			logger.debug(message);

		}

	}

	@Override

	protected boolean isInterceptorEnabled(final MethodInvocation invocation, final Log logger) {

		return true;

	}

}

services.oai.publisher.dnet.pagesize			=	100

services.oai.publisher.repo.name				=	D-NET Service for supporting Open Archive Initiative requests

services.oai.publisher.repo.email				=	bardi@isti.cnr.it

services.oai.publisher.repo.granularity			= 	YYYY-MM-DDThh:mm:ssZ

services.oai.publisher.repo.earliestdatestamp	=	2013-10-03T15:53:06Z

services.oai.publisher.baseUrl					=

#do we support deleted records? NO, TRANSIENT, PERSISTENT

services.oai.publisher.deletedrecords			=	TRANSIENT	

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws"

	xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing"

	xmlns:p="http://www.springframework.org/schema/p" xmlns:http="http://cxf.apache.org/transports/http/configuration"

	xmlns:t="http://dnetlib.eu/springbeans/t" xmlns:template="http://dnetlib.eu/springbeans/template"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd

    http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd 

    http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd 

    http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd 

    http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd 

    http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">

	<!-- beans -->

	<bean id="oaiProperties" class="eu.dnetlib.data.information.oai.publisher.OAIProperties"

		p:repoEmail="${services.oai.publisher.repo.email}" p:repoName="${services.oai.publisher.repo.name}"

		p:earliestDatestamp="${services.oai.publisher.repo.earliestdatestamp}"

		p:baseUrl="${services.oai.publisher.baseUrl}" p:deletedRecordSupport="${services.oai.publisher.deletedrecords}"

		p:dateGranularity="${services.oai.publisher.repo.granularity}" />

	<bean id="dnetOAICore"

		class="eu.dnetlib.data.information.oai.publisher.core.DNetOAICore"

		p:pageSize="${services.oai.publisher.dnet.pagesize}"/>

	<!-- Tracing -->

	<bean name="oaiControllerInterceptor"

		class="eu.dnetlib.data.information.oai.publisher.core.CoreInterceptor">

		<property name="enterMessage"

			value="ENTER: $[targetClassShortName].$[methodName]($[arguments])" />

		<property name="exitMessage"

			value="EXIT: $[targetClassShortName].$[methodName]() : $[invocationTime]ms" />

	</bean>

	<bean

		class="org.springframework.aop.framework.autoproxy.BeanNameAutoProxyCreator">

		<property name="beanNames" value="dnetOAICore" />

		<property name="proxyTargetClass" value="true" />

		<property name="interceptorNames">

			<list>

				<value>oaiControllerInterceptor</value>

			</list>

		</property>

	</bean>

</beans>

package eu.dnetlib.data.information.oai.publisher;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertFalse;

import java.text.Normalizer;

import org.apache.commons.lang.StringEscapeUtils;

import org.junit.Test;

public class PublisherMiscTest {

	@Test

	public void test() {

		String id = "NonavCreation.filmportal.de/DIF_NonAVCreation_EUROPA_TM & © Aardman Animations, LTD";

		String newId = StringEscapeUtils.escapeXml(id);

		assertEquals("NonavCreation.filmportal.de/DIF_NonAVCreation_EUROPA_TM & © Aardman Animations, LTD", newId);

		assertFalse(id.equals(newId));

	}

	@Test

	public void test2() {

		// Hochschulschriftenserver - Universität Frankfurt am Main

		String s = "Publikationenserver der Georg-August-Universität Göttingen";

		System.out.println("String to normalize: " + s);

		s = StringEscapeUtils.unescapeXml(s);

		System.out.println("unescaped: " + s);

		s = Normalizer.normalize(s, Normalizer.Form.NFD);

		System.out.println("normalized: " + s);

		// remove tilde, dots... over letters

		s = s.replaceAll("[\\p{InCombiningDiacriticalMarks}&&[^-_]]", "");

		// change punctuation into an underscore

		s = s.replaceAll("[\\p{Punct}&&[^-_]]", "_");

		// remove all non-word charcheters

		s = s.replaceAll("[\\W&&[^-_]]", "");

		System.out.println("Converted setSpec to: " + s);

	}

}

org.apache.cxf.Logger=org.apache.cxf.common.logging.Log4jLogger

log4j.rootLogger=WARN, CONSOLE

log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender

log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout

log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n

log4j.logger.eu.dnetlib=INFO

log4j.logger.eu.dnetlib.data=DEBUG