Project

General

Profile

« Previous | Next » 

Revision 50881

[maven-release-plugin] copy for tag cnr-data-information-oai-publisher-8.0.1

View differences:

modules/cnr-data-information-oai-publisher/tags/cnr-data-information-oai-publisher-8.0.1/deploy.info
1
{"type_source": "SVN", "goal": "package -U source:jar",
2
"url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/cnr-data-information-oai-publisher/trunk/",
3
"deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it",
4
"deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "cnr-data-information-oai-publisher"}
modules/cnr-data-information-oai-publisher/tags/cnr-data-information-oai-publisher-8.0.1/src/test/java/eu/dnetlib/data/information/oai/publisher/PublisherMiscTest.java
1
package eu.dnetlib.data.information.oai.publisher;
2

  
3
import java.text.Normalizer;
4

  
5
import org.apache.commons.lang3.StringEscapeUtils;
6
import org.junit.Ignore;
7
import org.junit.Test;
8

  
9
import static org.junit.Assert.assertEquals;
10
import static org.junit.Assert.assertFalse;
11

  
12
public class PublisherMiscTest {
13

  
14
	@Ignore
15
	@Test
16
	//FIXME: do we really need to change © into © ?
17
	public void test() {
18
		String id = "NonavCreation.filmportal.de/DIF_NonAVCreation_EUROPA_TM & © Aardman Animations, LTD";
19
		String newId = StringEscapeUtils.escapeXml11(id);
20
		assertEquals("NonavCreation.filmportal.de/DIF_NonAVCreation_EUROPA_TM & © Aardman Animations, LTD", newId);
21
		assertFalse(id.equals(newId));
22
	}
23

  
24
	@Test
25
	public void test2() {
26
		// Hochschulschriftenserver - Universität Frankfurt am Main
27
		String s = "Publikationenserver der Georg-August-Universität Göttingen";
28
		System.out.println("String to normalize: " + s);
29
		s = StringEscapeUtils.unescapeXml(s);
30
		System.out.println("unescaped: " + s);
31
		s = Normalizer.normalize(s, Normalizer.Form.NFD);
32
		System.out.println("normalized: " + s);
33
		// remove tilde, dots... over letters
34
		s = s.replaceAll("[\\p{InCombiningDiacriticalMarks}&&[^-_]]", "");
35
		// change punctuation into an underscore
36
		s = s.replaceAll("[\\p{Punct}&&[^-_]]", "_");
37
		// remove all non-word charcheters
38
		s = s.replaceAll("[\\W&&[^-_]]", "");
39
		System.out.println("Converted setSpec to: " + s);
40
	}
41

  
42
}
modules/cnr-data-information-oai-publisher/tags/cnr-data-information-oai-publisher-8.0.1/src/test/resources/log4j.properties
1
org.apache.cxf.Logger=org.apache.cxf.common.logging.Log4jLogger
2

  
3
log4j.rootLogger=WARN, CONSOLE
4
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
5
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
6
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
7

  
8
log4j.logger.eu.dnetlib=INFO
9
log4j.logger.eu.dnetlib.data=DEBUG
modules/cnr-data-information-oai-publisher/tags/cnr-data-information-oai-publisher-8.0.1/src/main/java/eu/dnetlib/data/information/oai/publisher/core/CoreInterceptor.java
1
package eu.dnetlib.data.information.oai.publisher.core;
2

  
3
import org.aopalliance.intercept.MethodInvocation;
4
import org.apache.commons.logging.Log;
5
import org.springframework.aop.interceptor.CustomizableTraceInterceptor;
6

  
7
/**
8
 * This class intercepts calls to the OAICore to measure its execution time. It uses Spring AOP.
9
 * 
10
 * @author alessia
11
 * 
12
 */
13
public class CoreInterceptor extends CustomizableTraceInterceptor {
14

  
15
	private static final long serialVersionUID = -9063818317778608736L;
16

  
17
	@Override
18
	protected void writeToLog(final Log logger, final String message, final Throwable ex) {
19
		if (ex != null) {
20
			logger.error(message, ex);
21
		} else {
22
			logger.debug(message);
23
		}
24
	}
25

  
26
	@Override
27
	protected boolean isInterceptorEnabled(final MethodInvocation invocation, final Log logger) {
28
		return true;
29
	}
30
}
modules/cnr-data-information-oai-publisher/tags/cnr-data-information-oai-publisher-8.0.1/src/main/java/eu/dnetlib/data/information/oai/publisher/core/DNetOAICore.java
1
package eu.dnetlib.data.information.oai.publisher.core;
2

  
3
import java.util.Iterator;
4
import java.util.List;
5
import javax.annotation.Resource;
6

  
7
import com.google.common.collect.Lists;
8
import eu.dnetlib.data.information.oai.publisher.*;
9
import eu.dnetlib.data.information.oai.publisher.conf.OAIConfigurationReader;
10
import eu.dnetlib.data.information.oai.publisher.info.ListDocumentsInfo;
11
import eu.dnetlib.data.information.oai.publisher.info.MDFInfo;
12
import eu.dnetlib.data.information.oai.publisher.info.RecordInfo;
13
import eu.dnetlib.data.information.oai.publisher.info.ResumptionTokenImpl;
14
import eu.dnetlib.data.oai.store.Cursor;
15
import eu.dnetlib.data.oai.store.PublisherStore;
16
import eu.dnetlib.data.oai.store.PublisherStoreDAO;
17
import eu.dnetlib.miscutils.functional.UnaryFunction;
18
import org.apache.commons.lang3.StringUtils;
19
import org.apache.commons.logging.Log;
20
import org.apache.commons.logging.LogFactory;
21

  
22
public class DNetOAICore extends AbstractOAICore {
23

  
24
	private static final Log log = LogFactory.getLog(DNetOAICore.class); // NOPMD by marko on 11/24/08 5:02 PM
25

  
26
	@Resource(name = "mongodbPublisherStoreDao")
27
	private PublisherStoreDAO<PublisherStore<Cursor>, Cursor> publisherStoreDAO;
28

  
29
	private String defaultDate = "2008-01-01T12:00:00Z";
30

  
31
	@Override
32
	protected RecordInfo getRecordById(final MDFInfo mdf, final String id) throws OaiPublisherException {
33
		PublisherStore<Cursor> store = this.publisherStoreDAO.getStoreFor(mdf.getPrefix(), getCurrentDBName());
34
		if (store == null)
35
			throw new OaiPublisherRuntimeException("Missing store for metadata prefix " + mdf.getPrefix() + ". Please check OAI publisher configuration.");
36
		RecordInfo record = null;
37
		if (StringUtils.isBlank(mdf.getTransformationRuleID())) {
38
			record = store.getRecord(id);
39
		} else {
40
			UnaryFunction<String, String> function = getLookupClient().getUnaryFunctionFromTDSRule(mdf.getTransformationRuleID());
41
			record = store.getRecord(id, function);
42
		}
43
		if (record != null) {
44
			record.setPrefix(mdf.getPrefix());
45
		}
46
		return record;
47
	}
48

  
49
	/**
50
	 * 
51
	 * {@inheritDoc}
52
	 * 
53
	 * @see eu.dnetlib.data.information.oai.publisher.core.AbstractOAICore#getDocuments(boolean, java.lang.String, java.lang.String,
54
	 *      java.lang.String, java.lang.String)
55
	 */
56
	@Override
57
	protected ListDocumentsInfo getDocuments(final boolean onlyIdentifiers, final String set, final String metadataPrefix, final String from, final String until)
58
			throws OaiPublisherException {
59
		MDFInfo mdf = obtainMDFInfo(metadataPrefix);
60
		boolean hasDateRange = StringUtils.isNotBlank(from) || StringUtils.isNotBlank(until);
61
		String query = this.generateQuery(mdf, set, from, until, hasDateRange);
62
		int total = this.countTotal(hasDateRange, query, set, mdf);
63
		log.debug("Total number of records: " + total);
64
		Cursor results = this.getCursor(query, onlyIdentifiers, mdf);
65
		ListDocumentsInfo res = this.prepareListDocumentsInfo(results, mdf, query, set, 0, total, hasDateRange);
66
		log.debug("Delivering " + res.getDocs().size() + " in a page");
67
		return res;
68
	}
69

  
70
	@Override
71
	protected ListDocumentsInfo getDocuments(final boolean onlyIdentifiers, final String resumptionToken) throws OaiPublisherException {
72
		ResumptionTokenImpl resToken = new ResumptionTokenImpl();
73
		resToken.deserialize(resumptionToken);
74

  
75
		log.debug(resToken.toString());
76

  
77
		MDFInfo mdf = obtainMDFInfo(resToken.getMetadataPrefix());
78
		String lastID = resToken.getLastObjIdentifier();
79
		String query = resToken.getQuery();
80
		String newQuery = "";
81
		if (StringUtils.isNotBlank(query)) {
82
			newQuery = query + " AND ";
83
		}
84
		newQuery += " _id > \"" + lastID + "\"";
85
		log.debug("NEW QUERY BECAUSE of resumptionToken: " + newQuery);
86
		int total = this.countTotal(resToken.hasDateRange(), query, resToken.getRequestedSet(), mdf);
87
		Cursor results = this.getCursor(newQuery, onlyIdentifiers, mdf);
88
		int oldCount = resToken.getnMaxElements();
89
		// if the number of records changed, then for sure we can invalidate the resumption token, unless we have a new total of -1 (date
90
		// range queries can't be counted for performance reasons)
91
		if ((total != -1) && (oldCount != total)) throw new BadResumptionTokenException(resumptionToken);
92

  
93
		ListDocumentsInfo res = this.prepareListDocumentsInfo(results, mdf, query, resToken.getRequestedSet(), resToken.getnRead(), resToken.getnMaxElements(),
94
				resToken.hasDateRange());
95
		res.setCursor(resToken.getnRead());
96
		return res;
97
	}
98

  
99
	protected ListDocumentsInfo prepareListDocumentsInfo(final Cursor results,
100
			final MDFInfo mdf,
101
			final String query,
102
			final String requestedSet,
103
			final int read,
104
			final int totalNumber,
105
			final boolean hasDateRange) throws OaiPublisherException {
106
		ListDocumentsInfo documentList = new ListDocumentsInfo();
107
		documentList.setnMaxElements(totalNumber);
108
		documentList.setMetadataPrefix(mdf.getPrefix());
109
		documentList.setCursor(0);
110
		if (documentList.getnMaxElements() == 0) throw new NoRecordsMatchException(OAIError.noRecordsMatch.getMessage());
111

  
112
		List<RecordInfo> theRecords = this.generateOAIRecords(mdf, requestedSet, results);
113
		documentList.setDocs(theRecords);
114

  
115
		if ((theRecords == null) || theRecords.isEmpty()) throw new NoRecordsMatchException("noRecordsMatch: 'documents' is null or empty");
116

  
117
		if ((documentList.getnMaxElements() > (read + theRecords.size())) || (documentList.getnMaxElements() == -1)) {
118
			String lastID = theRecords.get(theRecords.size() - 1).getInternalId();
119
			ResumptionTokenImpl nextToken = new ResumptionTokenImpl();
120
			nextToken.setDateRange(hasDateRange);
121
			nextToken.setLastObjIdentifier(lastID);
122
			nextToken.setMetadataPrefix(mdf.getPrefix());
123
			nextToken.setnMaxElements(totalNumber);
124
			nextToken.setnRead(read + theRecords.size());
125
			nextToken.setQuery(query);
126
			nextToken.setRequestedSet(requestedSet);
127
			documentList.setResumptionToken(nextToken);
128
		}
129

  
130
		return documentList;
131
	}
132

  
133
	protected Cursor getCursor(final String query, final boolean onlyIdentifiers, final MDFInfo mdfInfo) {
134
		PublisherStore<Cursor> store = this.publisherStoreDAO.getStore(mdfInfo.getSourceFormatName(), mdfInfo.getSourceFormatInterpretation(),
135
				mdfInfo.getSourceFormatLayout(), getCurrentDBName());
136
		if (store == null)
137
			throw new OaiPublisherRuntimeException("Missing store for metadata prefix " + mdfInfo.getPrefix() + ". Please check OAI publisher configuration.");
138
		Cursor results = null;
139
		if (StringUtils.isBlank(mdfInfo.getTransformationRuleID())) {
140
			results = store.getRecords(query, !onlyIdentifiers, pageSize);
141
		} else {
142
			UnaryFunction<String, String> function = getLookupClient().getUnaryFunctionFromTDSRule(mdfInfo.getTransformationRuleID());
143
			results = store.getRecords(query, function, !onlyIdentifiers, pageSize);
144
		}
145
		return results;
146
	}
147

  
148
	/**
149
	 * Generates the List of RecordInfo to be delivered.
150
	 *
151
	 * @param mdf
152
	 *            MDFInfo, the requested metadata format information.
153
	 * @param requestedSet
154
	 *            set specified in the request. It is blank if no set was requested.
155
	 * @param cursor
156
	 *            Cursor instance to use to get the records.
157
	 * @return List of RecordInfo instances
158
	 */
159
	protected List<RecordInfo> generateOAIRecords(final MDFInfo mdf, final String requestedSet, final Cursor cursor) {
160
		final List<RecordInfo> documents = Lists.newArrayList();
161
		Iterator<RecordInfo> cursorIterator = cursor.iterator();
162
		while (cursorIterator.hasNext()) {
163
			RecordInfo current = cursorIterator.next();
164
			current.addSetspec(requestedSet);
165
			current.setPrefix(mdf.getPrefix());
166
			documents.add(current);
167
		}
168
		return documents;
169
	}
170

  
171
	protected String generateQuery(final MDFInfo mdf, final String set, final String from, final String until, final boolean hasDateRange) {
172
		String datestampIndexName = OAIConfigurationReader.DATESTAMP_FIELD;
173

  
174
		String query = mdf.getBaseQuery();
175
		if (!StringUtils.isBlank(set)) {
176
			if (!StringUtils.isBlank(query)) {
177
				query += " AND ";
178
			}
179
			query += getSetCollection().getSetQuery(set, getCurrentDBName());
180
		}
181
		if (hasDateRange) {
182
			if (!StringUtils.isBlank(query)) {
183
				query += " AND ";
184
			}
185
			if ((from != null) && (until != null)) {
186
				query += datestampIndexName + " >= " + from + " AND " + datestampIndexName + " <= " + until;
187
			} else if (from != null) {
188
				query += datestampIndexName + " >= " + from;
189
			} else if (until != null) {
190
				query += datestampIndexName + " <= " + until;
191
			}
192
		}
193

  
194
		log.info("QUERY GENERATED: \n" + query);
195
		return query;
196
	}
197

  
198
	private int countTotal(final boolean hasDateRange, final String query, final String set, final MDFInfo mdFormat) {
199
		int total = 0;
200
		if (hasDateRange) {
201
			// Counting in the store by date ranges is too expensive and delays to much the response
202
			total = -1;
203
		} else {
204
			String theSet = set;
205
			if (StringUtils.isBlank(set)) {
206
				theSet = "ALL";
207
			}
208
			log.debug("SET::: " + theSet);
209
			total = getSetCollection().count(theSet, mdFormat.getPrefix(), getCurrentDBName());
210
		}
211
		return total;
212
	}
213

  
214
	public String getDefaultDate() {
215
		return defaultDate;
216
	}
217

  
218
	public void setDefaultDate(final String defaultDate) {
219
		this.defaultDate = defaultDate;
220
	}
221

  
222
	public PublisherStoreDAO<PublisherStore<Cursor>, Cursor> getPublisherStoreDAO() {
223
		return publisherStoreDAO;
224
	}
225

  
226
	public void setPublisherStoreDAO(final PublisherStoreDAO<PublisherStore<Cursor>, Cursor> publisherStoreDAO) {
227
		this.publisherStoreDAO = publisherStoreDAO;
228
	}
229

  
230
	public int getPageSize() {
231
		return pageSize;
232
	}
233

  
234
	public void setPageSize(final int pageSize) {
235
		this.pageSize = pageSize;
236
	}
237

  
238
}
modules/cnr-data-information-oai-publisher/tags/cnr-data-information-oai-publisher-8.0.1/src/main/resources/eu/dnetlib/data/information/oai/publisher/applicationContext-dnet-oaipublisher.properties
1
services.oai.publisher.dnet.pagesize			=	100
2
services.oai.publisher.repo.name				=	D-NET Service for supporting Open Archive Initiative requests
3
services.oai.publisher.repo.email				=	bardi@isti.cnr.it
4
services.oai.publisher.repo.granularity			= 	YYYY-MM-DDThh:mm:ssZ
5
services.oai.publisher.repo.earliestdatestamp	=	2013-10-03T15:53:06Z
6
services.oai.publisher.baseUrl					=
7
#do we support deleted records? NO, TRANSIENT, PERSISTENT
8
services.oai.publisher.deletedrecords			=	TRANSIENT	
9

  
modules/cnr-data-information-oai-publisher/tags/cnr-data-information-oai-publisher-8.0.1/src/main/resources/eu/dnetlib/data/information/oai/publisher/applicationContext-dnet-oaipublisher.xml
1
<?xml version="1.0" encoding="UTF-8"?>
2
<beans xmlns="http://www.springframework.org/schema/beans"
3
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws"
4
	xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing"
5
	xmlns:p="http://www.springframework.org/schema/p" xmlns:http="http://cxf.apache.org/transports/http/configuration"
6
	xmlns:t="http://dnetlib.eu/springbeans/t" xmlns:template="http://dnetlib.eu/springbeans/template"
7
	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
8
    http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd 
9
    http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd 
10
    http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd 
11
    http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd 
12
    http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">
13

  
14
	<!-- beans -->
15
	<bean id="oaiProperties" class="eu.dnetlib.data.information.oai.publisher.OAIProperties"
16
		p:repoEmail="${services.oai.publisher.repo.email}" p:repoName="${services.oai.publisher.repo.name}"
17
		p:earliestDatestamp="${services.oai.publisher.repo.earliestdatestamp}"
18
		p:baseUrl="${services.oai.publisher.baseUrl}" p:deletedRecordSupport="${services.oai.publisher.deletedrecords}"
19
		p:dateGranularity="${services.oai.publisher.repo.granularity}" />
20

  
21
	<bean id="dnetOAICore"
22
		class="eu.dnetlib.data.information.oai.publisher.core.DNetOAICore"
23
		p:pageSize="${services.oai.publisher.dnet.pagesize}"/>
24

  
25
	<!-- Tracing -->
26

  
27
	<bean name="oaiControllerInterceptor"
28
		class="eu.dnetlib.data.information.oai.publisher.core.CoreInterceptor">
29
		<property name="enterMessage"
30
			value="ENTER: $[targetClassShortName].$[methodName]($[arguments])" />
31
		<property name="exitMessage"
32
			value="EXIT: $[targetClassShortName].$[methodName]() : $[invocationTime]ms" />
33
	</bean>
34

  
35
	<bean
36
		class="org.springframework.aop.framework.autoproxy.BeanNameAutoProxyCreator">
37
		<property name="beanNames" value="dnetOAICore" />
38
		<property name="proxyTargetClass" value="true" />
39
		<property name="interceptorNames">
40
			<list>
41
				<value>oaiControllerInterceptor</value>
42
			</list>
43
		</property>
44
	</bean>
45

  
46
</beans>
modules/cnr-data-information-oai-publisher/tags/cnr-data-information-oai-publisher-8.0.1/pom.xml
1
<?xml version="1.0" encoding="UTF-8"?>
2
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
3
	<parent>
4
		<groupId>eu.dnetlib</groupId>
5
		<artifactId>dnet45-parent</artifactId>
6
		<version>1.0.0</version>
7
		<relativePath />
8
	</parent>
9
	<modelVersion>4.0.0</modelVersion>
10
	<groupId>eu.dnetlib</groupId>
11
	<artifactId>cnr-data-information-oai-publisher</artifactId>
12
	<packaging>jar</packaging>
13
	<version>8.0.1</version>
14
	<scm>
15
		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/cnr-data-information-oai-publisher/tags/cnr-data-information-oai-publisher-8.0.1</developerConnection>
16
	</scm>
17
	<dependencies>
18
		<dependency>
19
			<groupId>eu.dnetlib</groupId>
20
			<artifactId>cnr-data-information-oai-publisher-common</artifactId>
21
			<version>[6.0.0,7.0.0)</version>
22
		</dependency>
23
		<dependency>
24
			<groupId>eu.dnetlib</groupId>
25
			<artifactId>dnet-oai-store-service</artifactId>
26
			<version>[8.0.0,9.0.0)</version>
27
		</dependency>
28
		<dependency>
29
			<groupId>org.springframework</groupId>
30
			<artifactId>spring-aop</artifactId>
31
			<version>${spring.version}</version>
32
		</dependency>
33
		<dependency>
34
			<groupId>junit</groupId>
35
			<artifactId>junit</artifactId>
36
			<version>${junit.version}</version>
37
			<scope>test</scope>
38
		</dependency>
39

  
40
	</dependencies>
41
</project>

Also available in: Unified diff