Project

General

Profile

« Previous | Next » 

Revision 45138

codebase used to migrate to java8 the production system

View differences:

modules/dnet-oai-utils/trunk/src/main/resources/eu/dnetlib/data/information/oai/publisher/cache/ehcache.xml
1
<ehcache>
2

  
3
	<!-- Sets the path to the directory where cache .data files are created. 
4
		If the path is a Java System Property it is replaced by its value in the 
5
		running VM. The following properties are translated: user.home - User's home 
6
		directory user.dir - User's current working directory java.io.tmpdir - Default 
7
		temp file path -->
8
	<diskStore path="java.io.tmpdir" />
9

  
10

  
11
	<!--Default Cache configuration. These will applied to caches programmatically 
12
		created through the CacheManager. The following attributes are required for 
13
		defaultCache: maxInMemory - Sets the maximum number of objects that will 
14
		be created in memory eternal - Sets whether elements are eternal. If eternal, 
15
		timeouts are ignored and the element is never expired. timeToIdleSeconds 
16
		- Sets the time to idle for an element before it expires. i.e. The maximum 
17
		amount of time between accesses before an element expires Is only used if 
18
		the element is not eternal. Optional attribute. A value of 0 means that an 
19
		Element can idle for infinity timeToLiveSeconds - Sets the time to live for 
20
		an element before it expires. i.e. The maximum time between creation time 
21
		and when an element expires. Is only used if the element is not eternal. 
22
		overflowToDisk - Sets whether elements can overflow to disk when the in-memory 
23
		cache has reached the maxInMemory limit. -->
24

  
25
	<!-- <defaultCache maxElementsInMemory="1000" overflowToDisk="true" -->
26
	<!-- eternal="true" diskPersistent="false" timeToIdleSeconds="0" -->
27
	<!-- timeToLiveSeconds="0" memoryStoreEvictionPolicy="LRU" -->
28
	<!-- diskExpiryThreadIntervalSeconds="120" diskSpoolBufferSizeMB="5" /> -->
29

  
30

  
31
	<cache name="unaryFunctions" maxElementsInMemory="5" eternal="false"
32
		timeToIdleSeconds="500" timeToLiveSeconds="500" overflowToDisk="false" />
33

  
34
	<cache name="oaiDB" maxElementsInMemory="1" eternal="false"
35
		timeToIdleSeconds="1800" timeToLiveSeconds="1800" overflowToDisk="false" />
36

  
37
<!-- 	<cache name="publisherfields" maxElementsInMemory="30" eternal="false" -->
38
<!-- 		timeToIdleSeconds="1800" timeToLiveSeconds="1800" overflowToDisk="false" /> -->
39
		
40
<!-- 	<cache name="publisheridscheme" maxElementsInMemory="1" eternal="false" -->
41
<!-- 		timeToIdleSeconds="1800" timeToLiveSeconds="1800" overflowToDisk="false" /> -->
42
		
43
<!-- 	<cache name="publisheridnamespace" maxElementsInMemory="1" -->
44
<!-- 		eternal="false" timeToIdleSeconds="1800" timeToLiveSeconds="1800" -->
45
<!-- 		overflowToDisk="false" /> -->
46

  
47

  
48

  
49

  
50

  
51
</ehcache>
52

  
modules/dnet-oai-utils/trunk/pom.xml
1
<?xml version="1.0" encoding="UTF-8"?>
2
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
3
	<parent>
4
		<groupId>eu.dnetlib</groupId>
5
		<artifactId>dnet-parent</artifactId>
6
		<version>1.0.0</version>
7
		<relativePath />
8
	</parent>
9
	<modelVersion>4.0.0</modelVersion>
10
	<artifactId>dnet-oai-utils</artifactId>
11
	<packaging>jar</packaging>
12
	<version>3.3.3-SNAPSHOT</version>
13
	<scm>
14
		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-oai-utils/trunk</developerConnection>
15
	</scm>
16
	<dependencies>
17
		<dependency>
18
			<groupId>junit</groupId>
19
			<artifactId>junit</artifactId>
20
			<version>${junit.version}</version>
21
			<scope>test</scope>
22
		</dependency>
23
		<dependency>
24
			<groupId>eu.dnetlib</groupId>
25
			<artifactId>cnr-misc-utils</artifactId>
26
			<version>[1.0.0,2.0.0)</version>
27
		</dependency>
28
		<dependency>
29
			<groupId>com.google.guava</groupId>
30
			<artifactId>guava</artifactId>
31
			<version>${google.guava.version}</version>
32
		</dependency>
33
		<!-- Needed for locators -->
34
		<dependency>
35
			<groupId>eu.dnetlib</groupId>
36
			<artifactId>cnr-service-common</artifactId>
37
			<version>[2.1.0,3.0.0)</version>
38
		</dependency>
39

  
40
	</dependencies>
41
</project>
modules/dnet-oai-utils/tags/dnet-oai-utils-3.3.2/src/main/java/eu/dnetlib/data/information/oai/publisher/conf/OAIConfigurationStringReader.java
1
package eu.dnetlib.data.information.oai.publisher.conf;
2

  
3
import java.io.IOException;
4
import java.util.Collection;
5
import java.util.List;
6

  
7
import javax.annotation.Resource;
8

  
9
import org.apache.commons.logging.Log;
10
import org.apache.commons.logging.LogFactory;
11

  
12
import com.google.common.base.Function;
13
import com.google.common.base.Predicate;
14
import com.google.common.collect.Collections2;
15
import com.google.common.collect.Iterables;
16
import com.google.common.collect.Lists;
17

  
18
import eu.dnetlib.data.information.oai.publisher.OaiPublisherRuntimeException;
19
import eu.dnetlib.data.information.oai.publisher.PublisherField;
20
import eu.dnetlib.data.information.oai.publisher.info.MDFInfo;
21
import eu.dnetlib.data.information.oai.publisher.info.SetInfo;
22

  
23
/**
24
 * Instances of this class reads the OAI configuration from a string, which is the configuration profile passed in as a string.
25
 * 
26
 * @author alessia
27
 * 
28
 */
29
public class OAIConfigurationStringReader implements OAIConfigurationReader {
30

  
31
	private static final Log log = LogFactory.getLog(OAIConfigurationStringReader.class); // NOPMD by marko on 11/24/08 5:02 PM
32

  
33
	private OAIConfiguration oaiConfiguration;
34
	@Resource
35
	private OAIConfigurationParser configurationParser;
36

  
37
	public OAIConfigurationStringReader() {
38
		oaiConfiguration = null;
39
	}
40

  
41
	public OAIConfigurationStringReader(final String profile) {
42
		this.readConfiguration(profile);
43
	}
44

  
45
	public void readConfiguration(final String profile) {
46
		log.debug(profile);
47
		try {
48
			oaiConfiguration = configurationParser.getConfiguration(profile);
49
		} catch (IOException e) {
50
			throw new OaiPublisherRuntimeException("Could not read OAI configuration profile", e);
51
		}
52
	}
53

  
54
	public boolean isConfigurationLoaded() {
55
		return this.oaiConfiguration != null;
56
	}
57

  
58
	@Override
59
	public List<SetInfo> getSets() {
60
		if (isConfigurationLoaded()) return Lists.newArrayList(this.oaiConfiguration.getSetsMap().values());
61
		else throw new OaiPublisherRuntimeException("Configuration is not loaded");
62
	}
63

  
64
	@Override
65
	public List<SetInfo> getSets(final boolean onlyEnabled) {
66
		if (isConfigurationLoaded()) return Lists.newArrayList(Iterables.filter(this.oaiConfiguration.getSetsMap().values(), new Predicate<SetInfo>() {
67

  
68
			/**
69
			 * {@inheritDoc}
70
			 * 
71
			 * @see com.google.common.base.Predicate#apply(java.lang.Object)
72
			 */
73
			@Override
74
			public boolean apply(final SetInfo setInfo) {
75
				if (onlyEnabled) return setInfo.isEnabled();
76
				else return true;
77
			}
78

  
79
		}));
80
		else throw new OaiPublisherRuntimeException("Configuration is not loaded");
81
	}
82

  
83
	@Override
84
	public List<String> getSetSpecs() {
85
		if (isConfigurationLoaded()) return Lists.newArrayList(this.oaiConfiguration.getSetsMap().keySet());
86
		else throw new OaiPublisherRuntimeException("Configuration is not loaded");
87
	}
88

  
89
	@Override
90
	public List<String> getSetSpecs(final boolean onlyEnabled) {
91
		if (isConfigurationLoaded()) {
92
			if (!onlyEnabled) return this.getSetSpecs();
93
			List<SetInfo> enabled = this.getSets(true);
94
			return Lists.newArrayList(Iterables.transform(enabled, new Function<SetInfo, String>() {
95

  
96
				@Override
97
				public String apply(final SetInfo setInfo) {
98
					return setInfo.getSetSpec();
99
				}
100

  
101
			}));
102
		} else throw new OaiPublisherRuntimeException("Configuration is not loaded");
103
	}
104

  
105
	@Override
106
	public List<MDFInfo> getSourceMetadataFormats() {
107
		if (isConfigurationLoaded()) return Lists.newArrayList(this.oaiConfiguration.getSourcesMDF());
108
		else throw new OaiPublisherRuntimeException("Configuration is not loaded");
109
	}
110

  
111
	@Override
112
	public SetInfo getSetInfo(final String setSpec) {
113
		if (isConfigurationLoaded()) return this.oaiConfiguration.getSetsMap().get(setSpec);
114
		else throw new OaiPublisherRuntimeException("Configuration is not loaded");
115
	}
116

  
117
	@Override
118
	public List<MDFInfo> getMetadataFormatInfo() {
119
		if (isConfigurationLoaded()) return Lists.newArrayList(this.oaiConfiguration.getMdFormatsMap().values());
120
		else throw new OaiPublisherRuntimeException("Configuration is not loaded");
121
	}
122

  
123
	@Override
124
	public List<MDFInfo> getMetadataFormatInfo(final boolean onlyEnabled) {
125
		if (isConfigurationLoaded()) {
126
			if (!onlyEnabled) return this.getMetadataFormatInfo();
127
			else return Lists.newArrayList(Iterables.filter(this.oaiConfiguration.getMdFormatsMap().values(), new Predicate<MDFInfo>() {
128

  
129
				/**
130
				 * {@inheritDoc}
131
				 * 
132
				 * @see com.google.common.base.Predicate#apply(java.lang.Object)
133
				 */
134
				@Override
135
				public boolean apply(final MDFInfo mdfInfo) {
136
					if (onlyEnabled) return mdfInfo.isEnabled();
137
					else return true;
138
				}
139

  
140
			}));
141

  
142
		} else throw new OaiPublisherRuntimeException("Configuration is not loaded");
143
	}
144

  
145
	@Override
146
	public MDFInfo getMetadataFormatInfo(final String mdPrefix) {
147
		if (isConfigurationLoaded()) return this.oaiConfiguration.getMdFormatsMap().get(mdPrefix);
148
		else throw new OaiPublisherRuntimeException("Configuration is not loaded");
149
	}
150

  
151
	@Override
152
	public List<PublisherField> getFields() {
153
		if (isConfigurationLoaded()) return this.oaiConfiguration.getFields();
154
		else throw new OaiPublisherRuntimeException("Configuration is not loaded");
155
	}
156

  
157
	@Override
158
	public List<PublisherField> getFields(final String format, final String interpretation, final String layout) {
159
		if (isConfigurationLoaded()) return Lists.newArrayList(this.oaiConfiguration.getFieldsFor(format, layout, interpretation));
160
		else throw new OaiPublisherRuntimeException("Configuration is not loaded");
161
	}
162

  
163
	@Override
164
	public List<String> getFieldNames() {
165
		if (isConfigurationLoaded()) return this.oaiConfiguration.getFieldNames();
166
		else throw new OaiPublisherRuntimeException("Configuration is not loaded");
167
	}
168

  
169
	@Override
170
	public List<MDFInfo> getFormatsServedBy(final String sourceFormatName, final String sourceFormatLayout, final String sourceFormatInterpretation) {
171
		if (isConfigurationLoaded()) {
172
			Collection<MDFInfo> unfiltered = this.oaiConfiguration.getMdFormatsMap().values();
173
			return Lists.newArrayList(Collections2.filter(unfiltered, new Predicate<MDFInfo>() {
174

  
175
				@Override
176
				public boolean apply(final MDFInfo mdf) {
177
					return (mdf.getSourceFormatName() == sourceFormatName) && (mdf.getSourceFormatLayout() == sourceFormatLayout)
178
							&& (mdf.getSourceFormatInterpretation() == sourceFormatInterpretation);
179
				}
180

  
181
			}));
182

  
183
		} else throw new OaiPublisherRuntimeException("Configuration is not loaded");
184
	}
185

  
186
	@Override
187
	public String getIdScheme() {
188
		if (isConfigurationLoaded()) return this.oaiConfiguration.getIdScheme();
189
		else throw new OaiPublisherRuntimeException("Configuration is not loaded");
190
	}
191

  
192
	@Override
193
	public String getIdNamespace() {
194
		if (isConfigurationLoaded()) return this.oaiConfiguration.getIdNamespace();
195
		else throw new OaiPublisherRuntimeException("Configuration is not loaded");
196
	}
197

  
198
	public OAIConfiguration getOaiConfiguration() {
199
		return oaiConfiguration;
200
	}
201

  
202
	public void setOaiConfiguration(final OAIConfiguration oaiConfiguration) {
203
		this.oaiConfiguration = oaiConfiguration;
204
	}
205

  
206
	public OAIConfigurationParser getConfigurationParser() {
207
		return configurationParser;
208
	}
209

  
210
	public void setConfigurationParser(final OAIConfigurationParser configurationParser) {
211
		this.configurationParser = configurationParser;
212
	}
213

  
214
}
modules/dnet-oai-utils/trunk/src/main/java/eu/dnetlib/data/information/oai/utils/OAIParameterNames.java
1
package eu.dnetlib.data.information.oai.utils;
2

  
3
/**
4
 * Static class keeping track of env variable and job parameter names used by default by OAI Job nodes.
5
 * Created by alessia on 09/11/16.
6
 */
7
public class OAIParameterNames {
8

  
9
	public static String OAI_COLLECTON = "oai_collection";
10
	public static String OAI_DB = "oai_dbName";
11
	public static String OAI_FORMAT_NAME = "oai_format";
12
	public static String OAI_FORMAT_LAYOUT = "oai_layout";
13
	public static String OAI_FORMAT_INTERPRETATION = "oai_interpretation";
14
	public static String OAI_SOURCE = "oai_source";
15
	public static String OAI_INDEXES = "oai_indexFieldNames";
16
	public static String OAI_CONFIGURED_SETS_ONLY = "oai_configuredSetsOnly";
17
	public static String OAI_CONFIGURATION_PROFILE = "oai_configuration";
18
	public static String OAI_FEED_DATE = "oai_feedDate";
19
	public static String OAI_SKIP_DUPLICATES = "oai_skipDuplicates";
20
	public static String OAI_DUPLICATE_XPATH = "oai_duplicateXPath";
21
	public static String OAI_SYNC_EPR = "oai_syncEpr";
22
	public static String OAI_ALWAYS_NEW_RECORD= "oai_alwaysNewRecord";
23

  
24

  
25

  
26

  
27
}
modules/dnet-oai-utils/tags/dnet-oai-utils-3.3.2/src/main/java/eu/dnetlib/data/information/oai/publisher/conf/OAIConfigurationReader.java
1
package eu.dnetlib.data.information.oai.publisher.conf;
2

  
3
import java.util.List;
4

  
5
import eu.dnetlib.data.information.oai.publisher.PublisherField;
6
import eu.dnetlib.data.information.oai.publisher.info.MDFInfo;
7
import eu.dnetlib.data.information.oai.publisher.info.SetInfo;
8

  
9
public interface OAIConfigurationReader {
10

  
11
	/**
12
	 * Constant names of fields used by the publisher to store records in the OAIStores. If you want an index over them, you need to specify
13
	 * it in the OAI Publisher Configuration profile.
14
	 */
15
	public final static String ID_FIELD = "objIdentifier";
16
	public final static String DATESTAMP_FIELD = "datestamp";
17
	public final static String SET_FIELD = "set";
18
	public final static String DELETED_FIELD = "deleted";
19
	public final static String BODY_FIELD = "body";
20
	public final static String UPDATED_FIELD = "updated";
21
	public final static String LAST_COLLECTION_DATE_FIELD = "lastCollectionDate";
22

  
23
	public List<SetInfo> getSets();
24

  
25
	public List<SetInfo> getSets(final boolean onlyEnabled);
26

  
27
	public List<String> getSetSpecs();
28

  
29
	public List<String> getSetSpecs(final boolean onlyEnabled);
30

  
31
	public SetInfo getSetInfo(final String setSpec);
32

  
33
	public List<MDFInfo> getMetadataFormatInfo();
34

  
35
	public List<MDFInfo> getMetadataFormatInfo(final boolean onlyEnabled);
36

  
37
	public MDFInfo getMetadataFormatInfo(final String mdPrefix);
38

  
39
	public List<String> getFieldNames();
40

  
41
	public List<PublisherField> getFields();
42

  
43
	public List<PublisherField> getFields(final String format, final String interpretation, final String layout);
44

  
45
	/**
46
	 * Used to generate the OAI identifiers compliant to the protocol. See
47
	 * http://www.openarchives.org/OAI/openarchivesprotocol.html#UniqueIdentifier.
48
	 */
49
	public String getIdScheme();
50

  
51
	/**
52
	 * Used to generate the OAI identifiers compliant to the protocol. See
53
	 * http://www.openarchives.org/OAI/openarchivesprotocol.html#UniqueIdentifier.
54
	 */
55
	public String getIdNamespace();
56

  
57
	public List<MDFInfo> getSourceMetadataFormats();
58

  
59
	public List<MDFInfo> getFormatsServedBy(String sourceFormatName, String sourceFormatLayout, String sourceFormatInterpretation);
60

  
61
}
modules/dnet-oai-utils/trunk/src/test/java/eu/dnetlib/data/information/oai/publisher/conf/OAIConfigurationParserTest.java
1
package eu.dnetlib.data.information.oai.publisher.conf;
2

  
3
import static org.junit.Assert.assertEquals;
4
import static org.junit.Assert.assertNotNull;
5
import static org.junit.Assert.assertTrue;
6

  
7
import java.io.IOException;
8
import java.nio.charset.Charset;
9
import java.util.List;
10
import java.util.Map;
11

  
12
import org.apache.commons.io.IOUtils;
13
import org.apache.commons.logging.Log;
14
import org.apache.commons.logging.LogFactory;
15
import org.junit.Before;
16
import org.junit.Test;
17
import org.springframework.core.io.ClassPathResource;
18
import org.springframework.core.io.Resource;
19

  
20
import com.google.common.collect.ArrayListMultimap;
21
import com.google.common.collect.Lists;
22
import com.google.common.collect.Maps;
23
import com.google.common.collect.Multimap;
24

  
25
import eu.dnetlib.data.information.oai.publisher.PublisherField;
26
import eu.dnetlib.data.information.oai.publisher.info.MDFInfo;
27
import eu.dnetlib.data.information.oai.publisher.info.SetInfo;
28

  
29
public class OAIConfigurationParserTest {
30

  
31
	private static final Log log = LogFactory.getLog(OAIConfigurationParserTest.class); // NOPMD by marko on 11/24/08 5:02 PM
32

  
33
	/** Resource under test **/
34
	private OAIConfigurationParser parser;
35
	/** Configuration profile to parse **/
36
	private String configurationProfile;
37
	/** File path where to find the XML configuration profile **/
38
	private String pathToProfile = "eu/dnetlib/data/information/oai/publisher/conf/OAIPublisherConfiguration-1.xml";
39

  
40
	/** Expected configuration based on the profile available at the given pathToProfile. **/
41
	private OAIConfiguration expectedConfiguration;
42

  
43
	@Before
44
	public void setUp() throws Exception {
45

  
46
		// preparing the expected configuration that we will use to perform asserts on the result of parsing
47
		expectedConfiguration = new OAIConfiguration();
48
		Map<String, SetInfo> setsMap = Maps.newHashMap();
49
		setsMap.put("set1", new SetInfo("set1", "set number 1", "test set", "(set =\"aSet\")", true));
50
		expectedConfiguration.setSetsMap(setsMap);
51
		expectedConfiguration.setFieldNames(Lists.newArrayList("title", "objIdentifier", "set"));
52
		Map<String, MDFInfo> formatsMap = Maps.newHashMap();
53
		formatsMap.put("oai_dc", new MDFInfo("oai_dc", "SCHEMA", "NAMESPACE", "oai_dc", "store", "driver", "", true));
54
		formatsMap.put("hope", new MDFInfo("hope", "SCHEMA", "NAMESPACE", "hope", "L", "I", "country = IT", "tdsRule", true));
55
		expectedConfiguration.setMdFormatsMap(formatsMap);
56
		expectedConfiguration.setIdNamespace("eagle");
57
		List<PublisherField> fields = Lists.newArrayList();
58
		String oaidcKey = "oai_dc-store-driver";
59
		String hopeKey = "hope-L-I";
60

  
61
		PublisherField titleField = new PublisherField();
62
		titleField.setFieldName("title");
63
		titleField.setRepeatable(true);
64
		Multimap<String, String> titleSources = ArrayListMultimap.create();
65
		titleSources.put(oaidcKey, "//*[local-name() = 'title']");
66
		titleSources.put(oaidcKey, "//*[local-name() ='alternative']");
67
		titleSources.put(hopeKey, "//title");
68
		titleSources.put(hopeKey, "//titleinfo");
69
		titleField.setSources(titleSources);
70

  
71
		fields.add(titleField);
72

  
73
		PublisherField objIdField = new PublisherField();
74
		objIdField.setFieldName("objIdentifier");
75
		objIdField.setRepeatable(false);
76
		Multimap<String, String> objIdSources = ArrayListMultimap.create();
77
		objIdSources.put(oaidcKey, "//*[local-name() ='objIdentifier']");
78
		objIdField.setSources(objIdSources);
79

  
80
		fields.add(objIdField);
81

  
82
		PublisherField setField = new PublisherField();
83
		setField.setFieldName("set");
84
		setField.setRepeatable(true);
85
		Multimap<String, String> setSources = ArrayListMultimap.create();
86
		setSources.put(oaidcKey, "//*[local-name() ='repositoryId']");
87
		setField.setSources(setSources);
88

  
89
		fields.add(setField);
90

  
91
		expectedConfiguration.setFields(fields);
92
		Resource resource = new ClassPathResource(pathToProfile);
93
		// setting up the parser and the profile as a string
94
		parser = new OAIConfigurationParser();
95
		configurationProfile = IOUtils.toString(resource.getInputStream(), Charset.forName("UTF-8"));
96
		log.info("setUp completed");
97
	}
98

  
99
	@Test
100
	public void testSchemeIdNamespace() throws IOException {
101
		OAIConfiguration result = parser.getConfiguration(configurationProfile);
102
		System.out.println(result.getIdScheme());
103
		System.out.println(result.getIdNamespace());
104
	}
105

  
106
	@Test
107
	public void testGetConfiguration() throws IOException {
108
		OAIConfiguration result = parser.getConfiguration(configurationProfile);
109
		assertNotNull(result);
110
		// asserts on sets
111
		assertTrue(result.getSetsMap().containsKey("set1"));
112
		SetInfo expected = expectedConfiguration.getSetsMap().get("set1");
113
		SetInfo parsedSet = result.getSetsMap().get("set1");
114
		assertEquals(expected.getQuery(), parsedSet.getQuery());
115
		assertEquals(expected.getSetDescription(), parsedSet.getSetDescription());
116
		assertEquals(expected.getSetName(), parsedSet.getSetName());
117
		assertEquals(expected.getSetSpec(), parsedSet.getSetSpec());
118
		// asserts on indices
119
		assertTrue(result.getFieldNames().containsAll(expectedConfiguration.getFieldNames()));
120
		assertEquals(3, result.getFieldsFor("oai_dc", "store", "driver").size());
121
		assertEquals(1, result.getFieldsFor("hope", "L", "I").size());
122

  
123
		// asserts on metadata formats
124
		assertTrue(result.getMdFormatsMap().containsKey("hope"));
125
		assertTrue(result.getMdFormatsMap().containsKey("oai_dc"));
126
		MDFInfo expectedFormat = expectedConfiguration.getMdFormatsMap().get("hope");
127
		MDFInfo parsedFormat = result.getMdFormatsMap().get("hope");
128
		assertEquals(expectedFormat.getBaseQuery(), parsedFormat.getBaseQuery());
129
		assertEquals(expectedFormat.getNamespace(), parsedFormat.getNamespace());
130
		assertEquals(expectedFormat.getPrefix(), parsedFormat.getPrefix());
131
		assertEquals(expectedFormat.getSchema(), parsedFormat.getSchema());
132

  
133
		assertEquals(expectedFormat.getSourceFormatInterpretation(), parsedFormat.getSourceFormatInterpretation());
134
		assertEquals(expectedFormat.getSourceFormatLayout(), parsedFormat.getSourceFormatLayout());
135
		assertEquals(expectedFormat.getSourceFormatName(), parsedFormat.getSourceFormatName());
136
		assertEquals(expectedFormat.getTransformationRuleID(), parsedFormat.getTransformationRuleID());
137
		assertEquals(expectedConfiguration.getIdNamespace(), result.getIdNamespace());
138
		assertEquals("oai", result.getIdScheme());
139

  
140
	}
141

  
142
	@Test
143
	public void splitRSToken() {
144
		String token = "1434158|dli||100|5551ed26e4b0f1f9ff635bab|false|";
145
		String[] splitted = token.split("\\|");
146
		System.out.println(splitted.length);
147
		for (String s : splitted) {
148
			System.out.println(s);
149
		}
150
	}
151
}
modules/dnet-oai-utils/trunk/deploy.info
1
{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-oai-utils/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "dnet-oai-utils"}
modules/dnet-oai-utils/tags/dnet-oai-utils-3.3.2/src/main/java/eu/dnetlib/data/information/oai/utils/OAIParameterNames.java
1
package eu.dnetlib.data.information.oai.utils;
2

  
3
/**
4
 * Static class keeping track of env variable and job parameter names used by default by OAI Job nodes.
5
 * Created by alessia on 09/11/16.
6
 */
7
public class OAIParameterNames {
8

  
9
	public static String OAI_COLLECTON = "oai_collection";
10
	public static String OAI_DB = "oai_dbName";
11
	public static String OAI_FORMAT_NAME = "oai_format";
12
	public static String OAI_FORMAT_LAYOUT = "oai_layout";
13
	public static String OAI_FORMAT_INTERPRETATION = "oai_interpretation";
14
	public static String OAI_SOURCE = "oai_source";
15
	public static String OAI_INDEXES = "oai_indexFieldNames";
16
	public static String OAI_CONFIGURED_SETS_ONLY = "oai_configuredSetsOnly";
17
	public static String OAI_CONFIGURATION_PROFILE = "oai_configuration";
18
	public static String OAI_FEED_DATE = "oai_feedDate";
19
	public static String OAI_SKIP_DUPLICATES = "oai_skipDuplicates";
20
	public static String OAI_DUPLICATE_XPATH = "oai_duplicateXPath";
21
	public static String OAI_SYNC_EPR = "oai_syncEpr";
22
	public static String OAI_ALWAYS_NEW_RECORD= "oai_alwaysNewRecord";
23

  
24

  
25

  
26

  
27
}
modules/dnet-oai-utils/tags/dnet-oai-utils-3.3.2/src/main/java/eu/dnetlib/data/information/oai/publisher/conf/OAIConfigurationExistReader.java
1
package eu.dnetlib.data.information.oai.publisher.conf;
2

  
3
import java.util.Iterator;
4
import java.util.List;
5
import java.util.Set;
6

  
7
import org.springframework.beans.factory.annotation.Autowired;
8

  
9
import com.google.common.collect.ArrayListMultimap;
10
import com.google.common.collect.Lists;
11
import com.google.common.collect.Multimap;
12
import com.google.common.collect.Sets;
13

  
14
import eu.dnetlib.common.rmi.UnimplementedException;
15
import eu.dnetlib.data.information.oai.publisher.OaiPublisherRuntimeException;
16
import eu.dnetlib.data.information.oai.publisher.PublisherField;
17
import eu.dnetlib.data.information.oai.publisher.info.MDFInfo;
18
import eu.dnetlib.data.information.oai.publisher.info.SetInfo;
19
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
20
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
21
import eu.dnetlib.enabling.tools.SplittedQueryExecutor;
22

  
23
/**
24
 * Instances of this class read the configuration of the OAI publisher from the IS, by performing xquery on the Exist database.
25
 * 
26
 * @author alessia
27
 * 
28
 */
29
public class OAIConfigurationExistReader implements OAIConfigurationReader {
30

  
31
	@Autowired
32
	private SplittedQueryExecutor queryExecutor;
33
	private String xpathToIdScheme = "//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'OAIPublisherConfigurationDSResourceType']//CONFIGURATION/IDSCHEME/text()";
34
	private String xpathToIdNamespace = "//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'OAIPublisherConfigurationDSResourceType']//CONFIGURATION/IDNAMESPACE/text()";
35

  
36
	@Override
37
	public List<SetInfo> getSets() {
38
		final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION/OAISETS/OAISET "
39
				+ "return concat($x/spec, ':-:', $x/name, ':-:', $x//description , ':-:', $x//query, ':-:', $x/@enabled/string())";
40
		Iterable<SetInfo> sets = this.queryExecutor.query(SetInfo.class, query, ":-:");
41
		return Lists.newArrayList(sets);
42
	}
43

  
44
	@Override
45
	public List<SetInfo> getSets(final boolean onlyEnabled) {
46
		if (onlyEnabled) {
47
			final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION/OAISETS/OAISET "
48
					+ "where $x/@enabled/string() = 'true' "
49
					+ "return concat($x/spec, ':-:', $x/name, ':-:', $x//description , ':-:', $x//query, ':-:', $x/@enabled/string())";
50
			Iterable<SetInfo> sets = this.queryExecutor.query(SetInfo.class, query, ":-:");
51
			return Lists.newArrayList(sets);
52
		} else return this.getSets();
53
	}
54

  
55
	@Override
56
	public List<String> getSetSpecs() {
57
		final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION/OAISETS/OAISET "
58
				+ "return $x/spec/string() ";
59
		List<String> names = this.queryExecutor.performQuery(query);
60
		return names;
61
	}
62

  
63
	@Override
64
	public List<String> getSetSpecs(final boolean onlyEnabled) {
65
		final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION/OAISETS/OAISET "
66
				+ "where $x/@enabled/string() = '" + onlyEnabled + "' " + "return $x/spec/string() ";
67
		List<String> names = this.queryExecutor.performQuery(query);
68
		return names;
69
	}
70

  
71
	@Override
72
	public SetInfo getSetInfo(final String setSpec) {
73
		final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION/OAISETS/OAISET "
74
				+ "where $x/spec = '"
75
				+ setSpec
76
				+ "' "
77
				+ "return concat($x/spec, ':-:', $x/name, ':-:', $x//description , ':-:', $x//query, ':-:', $x/@enabled/string())";
78
		Iterable<SetInfo> sets = this.queryExecutor.query(SetInfo.class, query, ":-:");
79
		if (sets.iterator().hasNext()) return sets.iterator().next();
80
		else return null;
81
	}
82

  
83
	@Override
84
	public List<MDFInfo> getSourceMetadataFormats() {
85
		final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION//METADATAFORMAT "
86
				+ "return concat($x//SOURCE_METADATA_FORMAT/@name/string(), ':-:',  $x//SOURCE_METADATA_FORMAT/@layout/string(), ':-:', $x//SOURCE_METADATA_FORMAT/@interpretation/string())";
87

  
88
		List<String> res;
89
		try {
90
			res = this.queryExecutor.getServiceLocator().getService(ISLookUpService.class).quickSearchProfile(query);
91
		} catch (ISLookUpException e) {
92
			throw new OaiPublisherRuntimeException(e);
93
		}
94
		Set<MDFInfo> sources = Sets.newHashSet();
95
		for (String src : res) {
96
			String[] splitted = src.split(":-:");
97
			MDFInfo mdfInfo = new MDFInfo();
98
			mdfInfo.setSourceFormatName(splitted[0]);
99
			mdfInfo.setSourceFormatLayout(splitted[1]);
100
			mdfInfo.setSourceFormatInterpretation(splitted[2]);
101
			sources.add(mdfInfo);
102
		}
103
		return Lists.newArrayList(sources);
104
	}
105

  
106
	@Override
107
	public List<MDFInfo> getMetadataFormatInfo() {
108
		final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION//METADATAFORMAT "
109
				+ "return concat($x/@metadataPrefix/string(), ':-:', $x//SCHEMA , ':-:', $x//NAMESPACE,  "
110
				+ "':-:', $x//SOURCE_METADATA_FORMAT/@name/string(), ':-:',  $x//SOURCE_METADATA_FORMAT/@layout/string(), ':-:', $x//SOURCE_METADATA_FORMAT/@interpretation/string(), "
111
				+ "':-:', $x//BASE_QUERY, ':-:',  $x//TRANSFORMATION_RULE, ':-:', $x/@exportable/string() )";
112
		Iterable<MDFInfo> mdFormats = this.queryExecutor.query(MDFInfo.class, query, ":-:");
113
		return Lists.newArrayList(mdFormats);
114
	}
115

  
116
	@Override
117
	public List<MDFInfo> getMetadataFormatInfo(final boolean onlyEnabled) {
118
		if (onlyEnabled) {
119
			final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION//METADATAFORMAT "
120
					+ "where $x/@exportable/string() = 'true' "
121
					+ "return concat($x/@metadataPrefix/string(), ':-:', $x//SCHEMA , ':-:', $x//NAMESPACE,  "
122
					+ "':-:', $x//SOURCE_METADATA_FORMAT/@name/string(), ':-:',  $x//SOURCE_METADATA_FORMAT/@layout/string(), ':-:', $x//SOURCE_METADATA_FORMAT/@interpretation/string(), "
123
					+ "':-:', $x//BASE_QUERY, ':-:',  $x//TRANSFORMATION_RULE, ':-:', $x/@exportable/string() )";
124
			Iterable<MDFInfo> mdFormats = this.queryExecutor.query(MDFInfo.class, query, ":-:");
125
			return Lists.newArrayList(mdFormats);
126
		} else return getMetadataFormatInfo();
127
	}
128

  
129
	@Override
130
	public MDFInfo getMetadataFormatInfo(final String mdPrefix) {
131
		final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION//METADATAFORMAT "
132
				+ "where $x/@metadataPrefix/string()='"
133
				+ mdPrefix
134
				+ "' "
135
				+ "return concat($x/@metadataPrefix/string(), ':-:', $x//SCHEMA , ':-:', $x//NAMESPACE,  "
136
				+ "':-:', $x//SOURCE_METADATA_FORMAT/@name/string(), ':-:',  $x//SOURCE_METADATA_FORMAT/@layout/string(), ':-:', $x//SOURCE_METADATA_FORMAT/@interpretation/string(), "
137
				+ "':-:', $x//BASE_QUERY,':-:', $x//TRANSFORMATION_RULE, ':-:', $x/@exportable/string() )";
138
		Iterable<MDFInfo> mdFormats = this.queryExecutor.query(MDFInfo.class, query, ":-:");
139
		Iterator<MDFInfo> iterator = mdFormats.iterator();
140
		if (iterator.hasNext()) return iterator.next();
141
		else return null;
142
	}
143

  
144
	@Override
145
	public List<PublisherField> getFields() {
146
		throw new UnimplementedException();
147
	}
148

  
149
	@Override
150
	public List<PublisherField> getFields(final String format, final String interpretation, final String layout) {
151
		final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION//INDICES/INDEX "
152
				+ "return concat($x/@name, ':-:', $x/@repeatable, ':-:', string-join($x/SOURCE[@name='"
153
				+ format
154
				+ "' and @interpretation='"
155
				+ interpretation
156
				+ "' and @layout='" + layout + "']/@path, ':-:'))";
157
		List<String> res;
158
		try {
159
			res = this.queryExecutor.getServiceLocator().getService(ISLookUpService.class).quickSearchProfile(query);
160
		} catch (ISLookUpException e) {
161
			throw new OaiPublisherRuntimeException(e);
162
		}
163
		List<PublisherField> fields = Lists.newArrayList();
164
		for (String index : res) {
165
			String[] splitted = index.split(":-:");
166
			String indexName = splitted[0];
167
			String repeatable = splitted[1];
168
			PublisherField field = new PublisherField();
169
			field.setFieldName(indexName);
170
			field.setRepeatable(Boolean.valueOf(repeatable));
171
			Multimap<String, String> sources = ArrayListMultimap.create();
172
			String mdFormat = format + "-" + layout + "-" + interpretation;
173
			for (int i = 2; i < splitted.length; i++) {
174
				sources.put(mdFormat, splitted[i]);
175
			}
176
			field.setSources(sources);
177
			fields.add(field);
178
		}
179
		return fields;
180
	}
181

  
182
	@Override
183
	public List<String> getFieldNames() {
184
		String query = "//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'OAIPublisherConfigurationDSResourceType']//CONFIGURATION//INDICES/INDEX/@name/string()";
185
		try {
186
			return this.queryExecutor.getServiceLocator().getService(ISLookUpService.class).quickSearchProfile(query);
187
		} catch (ISLookUpException e) {
188
			throw new OaiPublisherRuntimeException(e);
189
		}
190
	}
191

  
192
	@Override
193
	public List<MDFInfo> getFormatsServedBy(final String sourceFormatName, final String sourceFormatLayout, final String sourceFormatInterpretation) {
194
		final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION//METADATAFORMAT[.//SOURCE_METADATA_FORMAT/@name = '"
195
				+ sourceFormatName
196
				+ "' and .//SOURCE_METADATA_FORMAT/@layout = '"
197
				+ sourceFormatLayout
198
				+ "' and .//SOURCE_METADATA_FORMAT/@interpretation = '"
199
				+ sourceFormatInterpretation
200
				+ "'] "
201
				+ "return concat($x/@metadataPrefix/string(), ':-:', $x//SCHEMA , ':-:', $x//NAMESPACE,  "
202
				+ "':-:', $x//SOURCE_METADATA_FORMAT/@name/string(), ':-:',  $x//SOURCE_METADATA_FORMAT/@layout/string(), ':-:', $x//SOURCE_METADATA_FORMAT/@interpretation/string(), "
203
				+ "':-:', $x//BASE_QUERY, ':-:',  $x//TRANSFORMATION_RULE, ':-:', $x/@exportable/string() )";
204
		Iterable<MDFInfo> mdFormats = this.queryExecutor.query(MDFInfo.class, query, ":-:");
205
		return Lists.newArrayList(mdFormats);
206
	}
207

  
208
	@Override
209
	public String getIdScheme() {
210
		try {
211
			return this.queryExecutor.getServiceLocator().getService(ISLookUpService.class).getResourceProfileByQuery(getXpathToIdScheme());
212
		} catch (ISLookUpException e) {
213
			throw new OaiPublisherRuntimeException(e);
214
		}
215
	}
216

  
217
	@Override
218
	public String getIdNamespace() {
219
		try {
220
			return this.queryExecutor.getServiceLocator().getService(ISLookUpService.class).getResourceProfileByQuery(getXpathToIdNamespace());
221
		} catch (ISLookUpException e) {
222
			throw new OaiPublisherRuntimeException(e);
223
		}
224
	}
225

  
226
	public SplittedQueryExecutor getQueryExecutor() {
227
		return queryExecutor;
228
	}
229

  
230
	public void setQueryExecutor(final SplittedQueryExecutor queryExecutor) {
231
		this.queryExecutor = queryExecutor;
232
	}
233

  
234
	public String getXpathToIdScheme() {
235
		return xpathToIdScheme;
236
	}
237

  
238
	public void setXpathToIdScheme(final String xpathToIdScheme) {
239
		this.xpathToIdScheme = xpathToIdScheme;
240
	}
241

  
242
	public String getXpathToIdNamespace() {
243
		return xpathToIdNamespace;
244
	}
245

  
246
	public void setXpathToIdNamespace(final String xpathToIdNamespace) {
247
		this.xpathToIdNamespace = xpathToIdNamespace;
248
	}
249

  
250
}
modules/dnet-oai-utils/tags/dnet-oai-utils-3.3.2/src/main/java/eu/dnetlib/data/information/oai/publisher/conf/OAIConfiguration.java
1
package eu.dnetlib.data.information.oai.publisher.conf;
2

  
3
import java.util.Collection;
4
import java.util.List;
5
import java.util.Map;
6
import java.util.Set;
7

  
8
import com.google.common.base.Joiner;
9
import com.google.common.base.Predicate;
10
import com.google.common.collect.Collections2;
11
import com.google.common.collect.Sets;
12

  
13
import eu.dnetlib.data.information.oai.publisher.PublisherField;
14
import eu.dnetlib.data.information.oai.publisher.info.MDFInfo;
15
import eu.dnetlib.data.information.oai.publisher.info.SetInfo;
16

  
17
/**
18
 * An instance of this class represents an OAI Configuration profile.
19
 * 
20
 * @author alessia
21
 * 
22
 */
23
public class OAIConfiguration {
24

  
25
	/**
26
	 * Map for OAI sets. Keys: setSpec, values: SetInfo instances.
27
	 */
28
	private Map<String, SetInfo> setsMap;
29

  
30
	/**
31
	 * Map for metadata formats. Keys: metadata prefix, values: MDFInfo instances.
32
	 */
33
	private Map<String, MDFInfo> mdFormatsMap;
34

  
35
	private List<PublisherField> fields;
36

  
37
	private List<String> fieldNames;
38

  
39
	/**
40
	 * Used to generate the OAI identifiers compliant to the protocol. See
41
	 * http://www.openarchives.org/OAI/openarchivesprotocol.html#UniqueIdentifier. See property services.oai.publisher.id.scheme.
42
	 */
43
	private String idScheme;
44
	/**
45
	 * Used to generate the OAI identifiers compliant to the protocol. See
46
	 * http://www.openarchives.org/OAI/openarchivesprotocol.html#UniqueIdentifier. See property services.oai.publisher.id.namespace.
47
	 */
48
	private String idNamespace;
49

  
50
	private Set<MDFInfo> sourcesMDF = Sets.newHashSet();
51

  
52
	public Collection<PublisherField> getFieldsFor(final String format, final String layout, final String interpretation) {
53
		final String mdRef = Joiner.on("-").join(format, layout, interpretation);
54
		return Collections2.filter(this.getFields(), new Predicate<PublisherField>() {
55

  
56
			@Override
57
			public boolean apply(final PublisherField theField) {
58
				if (theField.getSources() == null) return false;
59
				return theField.getSources().containsKey(mdRef);
60
			}
61

  
62
		});
63
	}
64

  
65
	public Map<String, SetInfo> getSetsMap() {
66
		return setsMap;
67
	}
68

  
69
	public void setSetsMap(final Map<String, SetInfo> setsMap) {
70
		this.setsMap = setsMap;
71
	}
72

  
73
	public Map<String, MDFInfo> getMdFormatsMap() {
74
		return mdFormatsMap;
75
	}
76

  
77
	public void setMdFormatsMap(final Map<String, MDFInfo> mdFormatsMap) {
78
		this.mdFormatsMap = mdFormatsMap;
79
	}
80

  
81
	public List<PublisherField> getFields() {
82
		return fields;
83
	}
84

  
85
	public void setFields(final List<PublisherField> fields) {
86
		this.fields = fields;
87
	}
88

  
89
	public List<String> getFieldNames() {
90
		return fieldNames;
91
	}
92

  
93
	public void setFieldNames(final List<String> fieldNames) {
94
		this.fieldNames = fieldNames;
95
	}
96

  
97
	public Set<MDFInfo> getSourcesMDF() {
98
		return sourcesMDF;
99
	}
100

  
101
	public void setSourcesMDF(final Set<MDFInfo> sourcesMDF) {
102
		this.sourcesMDF = sourcesMDF;
103
	}
104

  
105
	public String getIdScheme() {
106
		return idScheme;
107
	}
108

  
109
	public void setIdScheme(final String idScheme) {
110
		this.idScheme = idScheme;
111
	}
112

  
113
	public String getIdNamespace() {
114
		return idNamespace;
115
	}
116

  
117
	public void setIdNamespace(final String idNamespace) {
118
		this.idNamespace = idNamespace;
119
	}
120

  
121
}
modules/dnet-oai-utils/tags/dnet-oai-utils-3.3.2/src/main/java/eu/dnetlib/data/information/oai/publisher/BadResumptionTokenException.java
1
package eu.dnetlib.data.information.oai.publisher;
2

  
3
public class BadResumptionTokenException extends OaiPublisherException {
4

  
5
	private static final long serialVersionUID = 1721833922974061626L;
6

  
7
	public BadResumptionTokenException(final String message) {
8
		super(message);
9
	}
10

  
11
}
modules/dnet-oai-utils/tags/dnet-oai-utils-3.3.2/src/main/java/eu/dnetlib/data/information/oai/publisher/info/RecordInfo.java
1
package eu.dnetlib.data.information.oai.publisher.info;
2

  
3
import java.text.SimpleDateFormat;
4
import java.util.Date;
5
import java.util.Set;
6
import java.util.TimeZone;
7

  
8
import com.google.common.collect.Sets;
9

  
10
public class RecordInfo {
11

  
12
	private String prefix = "";
13
	private String identifier = "";
14
	private Date datestamp = null;
15
	private Set<String> setspecs = Sets.newHashSet();
16
	private String metadata = "";
17
	private String internalId = "";
18
	private boolean deleted;
19
	private String provenance = "";
20

  
21
	public RecordInfo() {}
22

  
23
	public RecordInfo(final String prefix, final String identifier, final Date datestamp, final String setspec, final String metadata, final boolean deleted) {
24
		this(prefix, identifier, datestamp, Sets.newHashSet(setspec), metadata, deleted);
25
	}
26

  
27
	public RecordInfo(final String prefix, final String identifier, final Date datestamp, final Set<String> setspecs, final String metadata,
28
			final boolean deleted) {
29
		super();
30
		this.prefix = prefix;
31
		this.identifier = identifier;
32
		this.datestamp = datestamp;
33
		this.setspecs = setspecs;
34
		this.metadata = metadata;
35
		this.deleted = deleted;
36
	}
37

  
38
	public boolean addSetspec(final String setspec) {
39
		if (!setspecs.contains(setspec)) {
40
			setspecs.add(setspec);
41
			return true;
42
		} else return false;
43
	}
44

  
45
	public String getPrefix() {
46
		return prefix;
47
	}
48

  
49
	public void setPrefix(final String prefix) {
50
		this.prefix = prefix;
51
	}
52

  
53
	public String getIdentifier() {
54
		return identifier;
55
	}
56

  
57
	public void setIdentifier(final String identifier) {
58
		this.identifier = identifier;
59
	}
60

  
61
	public Date getDatestamp() {
62
		return datestamp;
63
	}
64

  
65
	public void setDatestamp(final Date datestamp) {
66
		this.datestamp = datestamp;
67
	}
68

  
69
	public Set<String> getSetspecs() {
70
		return setspecs;
71
	}
72

  
73
	public void setSetspecs(final Set<String> setspecs) {
74
		this.setspecs = setspecs;
75
	}
76

  
77
	public String getMetadata() {
78
		return metadata;
79
	}
80

  
81
	public void setMetadata(final String metadata) {
82
		this.metadata = metadata;
83
	}
84

  
85
	/**
86
	 * 
87
	 * @return the date formatted in ISO8601, UTC with format YYYY-MM-DDThh:mm:ssZ, where Z is the time zone.
88
	 */
89
	public String getNormalizedDatestamp() {
90
		SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");
91
		formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
92
		String date = formatter.format(datestamp);
93
		return date.replace("+0000", "Z");
94
	}
95

  
96
	public String getInternalId() {
97
		return internalId;
98
	}
99

  
100
	public void setInternalId(final String internalId) {
101
		this.internalId = internalId;
102
	}
103

  
104
	public boolean isDeleted() {
105
		return deleted;
106
	}
107

  
108
	public void setDeleted(final boolean deleted) {
109
		this.deleted = deleted;
110
	}
111

  
112
	public String getProvenance() {
113
		return provenance;
114
	}
115

  
116
	public void setProvenance(String provenance) {
117
		this.provenance = provenance;
118
	}
119

  
120
}
modules/dnet-oai-utils/trunk/src/main/java/eu/dnetlib/data/information/oai/publisher/conf/ISLookUpClient.java
1
package eu.dnetlib.data.information.oai.publisher.conf;
2

  
3
import java.util.List;
4
import javax.annotation.Resource;
5

  
6
import com.google.common.collect.Lists;
7
import eu.dnetlib.data.information.oai.publisher.OaiPublisherException;
8
import eu.dnetlib.data.information.oai.publisher.OaiPublisherRuntimeException;
9
import eu.dnetlib.data.information.oai.publisher.info.MDFInfo;
10
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpDocumentNotFoundException;
11
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
12
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
13
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
14
import eu.dnetlib.miscutils.functional.IdentityFunction;
15
import eu.dnetlib.miscutils.functional.UnaryFunction;
16
import eu.dnetlib.miscutils.functional.xml.ApplyXslt;
17
import org.apache.commons.lang.StringUtils;
18
import org.apache.commons.logging.Log;
19
import org.apache.commons.logging.LogFactory;
20
import org.dom4j.DocumentException;
21
import org.springframework.beans.factory.annotation.Value;
22
import org.springframework.cache.annotation.CacheEvict;
23
import org.springframework.cache.annotation.Cacheable;
24

  
25
/**
26
 * Helper class for common queries to the IS.
27
 * 
28
 * @author alessia
29
 * 
30
 */
31
public class ISLookUpClient {
32

  
33
	private static final Log log = LogFactory.getLog(ISLookUpClient.class); // NOPMD by marko on 11/24/08 5:02 PM
34

  
35
	@Value("${services.oai.publisher.db.xquery}")
36
	private String xpathToCurrentDB;
37

  
38
	/**
39
	 * lookup service.
40
	 */
41
	@Resource
42
	private UniqueServiceLocator serviceLocator;
43

  
44
	@Resource
45
	private OAIConfigurationExistReader configuration;
46

  
47
	public List<MDFInfo> listMetadataFormats(final boolean onlyEnabled) throws ISLookUpException, DocumentException, OaiPublisherException {
48
		return Lists.newArrayList(this.configuration.getMetadataFormatInfo(onlyEnabled));
49
	}
50

  
51
	@Cacheable(value = "unaryFunctions", key = "#tdsRuleId")
52
	public UnaryFunction<String, String> getUnaryFunctionFromTDSRule(final String tdsRuleId) {
53
		log.fatal("Not using cache to get TDSRule " + tdsRuleId);
54
		final String queryCodeRule = "//RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value='" + tdsRuleId
55
				+ "']/BODY/CONFIGURATION/SCRIPT/CODE/*[local-name()='stylesheet']";
56
		String xsltCode = null;
57
		try {
58
			xsltCode = this.serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(queryCodeRule);
59
		} catch (ISLookUpDocumentNotFoundException e) {
60
			log.fatal("Unexisting TDSRule profile with identifier " + tdsRuleId);
61
			throw new OaiPublisherRuntimeException(e);
62
		} catch (ISLookUpException e) {
63
			throw new OaiPublisherRuntimeException(e);
64
		}
65
		if (StringUtils.isBlank(xsltCode)) {
66
			log.warn("Unexpected blank stylesheet in TDSRule profile with id: " + tdsRuleId + ". Returning identity function.");
67
			return new IdentityFunction<String>();
68
		} else return new ApplyXslt(xsltCode);
69
	}
70

  
71
	@Cacheable("oaiDB")
72
	public String getCurrentDB() throws ISLookUpDocumentNotFoundException, ISLookUpException {
73
		log.fatal("Not using cache to get the current OAI db");
74
		String currentDBName = this.serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xpathToCurrentDB);
75
		log.info("Current OAI DB:" + currentDBName);
76
		return currentDBName;
77
	}
78

  
79
	@CacheEvict(value = { "unaryFunctions", "oaiDB" }, allEntries = true, beforeInvocation = true)
80
	public void evictCaches() {
81
		log.info("Evicted caches oaiDB and unaryFunctions");
82
	}
83

  
84
	public OAIConfigurationExistReader getConfiguration() {
85
		return configuration;
86
	}
87

  
88
	public void setConfiguration(final OAIConfigurationExistReader configuration) {
89
		this.configuration = configuration;
90
	}
91

  
92
}
modules/dnet-oai-utils/trunk/src/main/java/eu/dnetlib/data/information/oai/publisher/conf/OAIConfigurationExistReader.java
1
package eu.dnetlib.data.information.oai.publisher.conf;
2

  
3
import java.util.Iterator;
4
import java.util.List;
5
import java.util.Set;
6

  
7
import org.springframework.beans.factory.annotation.Autowired;
8

  
9
import com.google.common.collect.ArrayListMultimap;
10
import com.google.common.collect.Lists;
11
import com.google.common.collect.Multimap;
12
import com.google.common.collect.Sets;
13

  
14
import eu.dnetlib.common.rmi.UnimplementedException;
15
import eu.dnetlib.data.information.oai.publisher.OaiPublisherRuntimeException;
16
import eu.dnetlib.data.information.oai.publisher.PublisherField;
17
import eu.dnetlib.data.information.oai.publisher.info.MDFInfo;
18
import eu.dnetlib.data.information.oai.publisher.info.SetInfo;
19
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
20
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
21
import eu.dnetlib.enabling.tools.SplittedQueryExecutor;
22

  
23
/**
24
 * Instances of this class read the configuration of the OAI publisher from the IS, by performing xquery on the Exist database.
25
 * 
26
 * @author alessia
27
 * 
28
 */
29
public class OAIConfigurationExistReader implements OAIConfigurationReader {
30

  
31
	@Autowired
32
	private SplittedQueryExecutor queryExecutor;
33
	private String xpathToIdScheme = "//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'OAIPublisherConfigurationDSResourceType']//CONFIGURATION/IDSCHEME/text()";
34
	private String xpathToIdNamespace = "//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'OAIPublisherConfigurationDSResourceType']//CONFIGURATION/IDNAMESPACE/text()";
35

  
36
	@Override
37
	public List<SetInfo> getSets() {
38
		final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION/OAISETS/OAISET "
39
				+ "return concat($x/spec, ':-:', $x/name, ':-:', $x//description , ':-:', $x//query, ':-:', $x/@enabled/string())";
40
		Iterable<SetInfo> sets = this.queryExecutor.query(SetInfo.class, query, ":-:");
41
		return Lists.newArrayList(sets);
42
	}
43

  
44
	@Override
45
	public List<SetInfo> getSets(final boolean onlyEnabled) {
46
		if (onlyEnabled) {
47
			final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION/OAISETS/OAISET "
48
					+ "where $x/@enabled/string() = 'true' "
49
					+ "return concat($x/spec, ':-:', $x/name, ':-:', $x//description , ':-:', $x//query, ':-:', $x/@enabled/string())";
50
			Iterable<SetInfo> sets = this.queryExecutor.query(SetInfo.class, query, ":-:");
51
			return Lists.newArrayList(sets);
52
		} else return this.getSets();
53
	}
54

  
55
	@Override
56
	public List<String> getSetSpecs() {
57
		final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION/OAISETS/OAISET "
58
				+ "return $x/spec/string() ";
59
		List<String> names = this.queryExecutor.performQuery(query);
60
		return names;
61
	}
62

  
63
	@Override
64
	public List<String> getSetSpecs(final boolean onlyEnabled) {
65
		final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION/OAISETS/OAISET "
66
				+ "where $x/@enabled/string() = '" + onlyEnabled + "' " + "return $x/spec/string() ";
67
		List<String> names = this.queryExecutor.performQuery(query);
68
		return names;
69
	}
70

  
71
	@Override
72
	public SetInfo getSetInfo(final String setSpec) {
73
		final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION/OAISETS/OAISET "
74
				+ "where $x/spec = '"
75
				+ setSpec
76
				+ "' "
77
				+ "return concat($x/spec, ':-:', $x/name, ':-:', $x//description , ':-:', $x//query, ':-:', $x/@enabled/string())";
78
		Iterable<SetInfo> sets = this.queryExecutor.query(SetInfo.class, query, ":-:");
79
		if (sets.iterator().hasNext()) return sets.iterator().next();
80
		else return null;
81
	}
82

  
83
	@Override
84
	public List<MDFInfo> getSourceMetadataFormats() {
85
		final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION//METADATAFORMAT "
86
				+ "return concat($x//SOURCE_METADATA_FORMAT/@name/string(), ':-:',  $x//SOURCE_METADATA_FORMAT/@layout/string(), ':-:', $x//SOURCE_METADATA_FORMAT/@interpretation/string())";
87

  
88
		List<String> res;
89
		try {
90
			res = this.queryExecutor.getServiceLocator().getService(ISLookUpService.class).quickSearchProfile(query);
91
		} catch (ISLookUpException e) {
92
			throw new OaiPublisherRuntimeException(e);
93
		}
94
		Set<MDFInfo> sources = Sets.newHashSet();
95
		for (String src : res) {
96
			String[] splitted = src.split(":-:");
97
			MDFInfo mdfInfo = new MDFInfo();
98
			mdfInfo.setSourceFormatName(splitted[0]);
99
			mdfInfo.setSourceFormatLayout(splitted[1]);
100
			mdfInfo.setSourceFormatInterpretation(splitted[2]);
101
			sources.add(mdfInfo);
102
		}
103
		return Lists.newArrayList(sources);
104
	}
105

  
106
	@Override
107
	public List<MDFInfo> getMetadataFormatInfo() {
108
		final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION//METADATAFORMAT "
109
				+ "return concat($x/@metadataPrefix/string(), ':-:', $x//SCHEMA , ':-:', $x//NAMESPACE,  "
110
				+ "':-:', $x//SOURCE_METADATA_FORMAT/@name/string(), ':-:',  $x//SOURCE_METADATA_FORMAT/@layout/string(), ':-:', $x//SOURCE_METADATA_FORMAT/@interpretation/string(), "
111
				+ "':-:', $x//BASE_QUERY, ':-:',  $x//TRANSFORMATION_RULE, ':-:', $x/@exportable/string() )";
112
		Iterable<MDFInfo> mdFormats = this.queryExecutor.query(MDFInfo.class, query, ":-:");
113
		return Lists.newArrayList(mdFormats);
114
	}
115

  
116
	@Override
117
	public List<MDFInfo> getMetadataFormatInfo(final boolean onlyEnabled) {
118
		if (onlyEnabled) {
119
			final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION//METADATAFORMAT "
120
					+ "where $x/@exportable/string() = 'true' "
121
					+ "return concat($x/@metadataPrefix/string(), ':-:', $x//SCHEMA , ':-:', $x//NAMESPACE,  "
122
					+ "':-:', $x//SOURCE_METADATA_FORMAT/@name/string(), ':-:',  $x//SOURCE_METADATA_FORMAT/@layout/string(), ':-:', $x//SOURCE_METADATA_FORMAT/@interpretation/string(), "
123
					+ "':-:', $x//BASE_QUERY, ':-:',  $x//TRANSFORMATION_RULE, ':-:', $x/@exportable/string() )";
124
			Iterable<MDFInfo> mdFormats = this.queryExecutor.query(MDFInfo.class, query, ":-:");
125
			return Lists.newArrayList(mdFormats);
126
		} else return getMetadataFormatInfo();
127
	}
128

  
129
	@Override
130
	public MDFInfo getMetadataFormatInfo(final String mdPrefix) {
131
		final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION//METADATAFORMAT "
132
				+ "where $x/@metadataPrefix/string()='"
133
				+ mdPrefix
134
				+ "' "
135
				+ "return concat($x/@metadataPrefix/string(), ':-:', $x//SCHEMA , ':-:', $x//NAMESPACE,  "
136
				+ "':-:', $x//SOURCE_METADATA_FORMAT/@name/string(), ':-:',  $x//SOURCE_METADATA_FORMAT/@layout/string(), ':-:', $x//SOURCE_METADATA_FORMAT/@interpretation/string(), "
137
				+ "':-:', $x//BASE_QUERY,':-:', $x//TRANSFORMATION_RULE, ':-:', $x/@exportable/string() )";
138
		Iterable<MDFInfo> mdFormats = this.queryExecutor.query(MDFInfo.class, query, ":-:");
139
		Iterator<MDFInfo> iterator = mdFormats.iterator();
140
		if (iterator.hasNext()) return iterator.next();
141
		else return null;
142
	}
143

  
144
	@Override
145
	public List<PublisherField> getFields() {
146
		throw new UnimplementedException();
147
	}
148

  
149
	@Override
150
	public List<PublisherField> getFields(final String format, final String interpretation, final String layout) {
151
		final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION//INDICES/INDEX "
152
				+ "return concat($x/@name, ':-:', $x/@repeatable, ':-:', string-join($x/SOURCE[@name='"
153
				+ format
154
				+ "' and @interpretation='"
155
				+ interpretation
156
				+ "' and @layout='" + layout + "']/@path, ':-:'))";
157
		List<String> res;
158
		try {
159
			res = this.queryExecutor.getServiceLocator().getService(ISLookUpService.class).quickSearchProfile(query);
160
		} catch (ISLookUpException e) {
161
			throw new OaiPublisherRuntimeException(e);
162
		}
163
		List<PublisherField> fields = Lists.newArrayList();
164
		for (String index : res) {
165
			String[] splitted = index.split(":-:");
166
			String indexName = splitted[0];
167
			String repeatable = splitted[1];
168
			PublisherField field = new PublisherField();
169
			field.setFieldName(indexName);
170
			field.setRepeatable(Boolean.valueOf(repeatable));
171
			Multimap<String, String> sources = ArrayListMultimap.create();
172
			String mdFormat = format + "-" + layout + "-" + interpretation;
173
			for (int i = 2; i < splitted.length; i++) {
174
				sources.put(mdFormat, splitted[i]);
175
			}
176
			field.setSources(sources);
177
			fields.add(field);
178
		}
179
		return fields;
180
	}
181

  
182
	@Override
183
	public List<String> getFieldNames() {
184
		String query = "//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'OAIPublisherConfigurationDSResourceType']//CONFIGURATION//INDICES/INDEX/@name/string()";
185
		try {
186
			return this.queryExecutor.getServiceLocator().getService(ISLookUpService.class).quickSearchProfile(query);
187
		} catch (ISLookUpException e) {
188
			throw new OaiPublisherRuntimeException(e);
189
		}
190
	}
191

  
192
	@Override
193
	public List<MDFInfo> getFormatsServedBy(final String sourceFormatName, final String sourceFormatLayout, final String sourceFormatInterpretation) {
194
		final String query = "for $x in collection('/db/DRIVER/OAIPublisherConfigurationDSResources/OAIPublisherConfigurationDSResourceType')//CONFIGURATION//METADATAFORMAT[.//SOURCE_METADATA_FORMAT/@name = '"
195
				+ sourceFormatName
196
				+ "' and .//SOURCE_METADATA_FORMAT/@layout = '"
197
				+ sourceFormatLayout
198
				+ "' and .//SOURCE_METADATA_FORMAT/@interpretation = '"
199
				+ sourceFormatInterpretation
200
				+ "'] "
201
				+ "return concat($x/@metadataPrefix/string(), ':-:', $x//SCHEMA , ':-:', $x//NAMESPACE,  "
202
				+ "':-:', $x//SOURCE_METADATA_FORMAT/@name/string(), ':-:',  $x//SOURCE_METADATA_FORMAT/@layout/string(), ':-:', $x//SOURCE_METADATA_FORMAT/@interpretation/string(), "
203
				+ "':-:', $x//BASE_QUERY, ':-:',  $x//TRANSFORMATION_RULE, ':-:', $x/@exportable/string() )";
204
		Iterable<MDFInfo> mdFormats = this.queryExecutor.query(MDFInfo.class, query, ":-:");
205
		return Lists.newArrayList(mdFormats);
206
	}
207

  
208
	@Override
209
	public String getIdScheme() {
210
		try {
211
			return this.queryExecutor.getServiceLocator().getService(ISLookUpService.class).getResourceProfileByQuery(getXpathToIdScheme());
212
		} catch (ISLookUpException e) {
213
			throw new OaiPublisherRuntimeException(e);
214
		}
215
	}
216

  
217
	@Override
218
	public String getIdNamespace() {
219
		try {
220
			return this.queryExecutor.getServiceLocator().getService(ISLookUpService.class).getResourceProfileByQuery(getXpathToIdNamespace());
221
		} catch (ISLookUpException e) {
222
			throw new OaiPublisherRuntimeException(e);
223
		}
224
	}
225

  
226
	public SplittedQueryExecutor getQueryExecutor() {
227
		return queryExecutor;
228
	}
229

  
230
	public void setQueryExecutor(final SplittedQueryExecutor queryExecutor) {
231
		this.queryExecutor = queryExecutor;
232
	}
233

  
234
	public String getXpathToIdScheme() {
235
		return xpathToIdScheme;
236
	}
237

  
238
	public void setXpathToIdScheme(final String xpathToIdScheme) {
239
		this.xpathToIdScheme = xpathToIdScheme;
240
	}
241

  
242
	public String getXpathToIdNamespace() {
243
		return xpathToIdNamespace;
244
	}
245

  
246
	public void setXpathToIdNamespace(final String xpathToIdNamespace) {
247
		this.xpathToIdNamespace = xpathToIdNamespace;
248
	}
249

  
250
}
modules/dnet-oai-utils/trunk/src/main/java/eu/dnetlib/data/information/oai/publisher/conf/OAIConfigurationStringReader.java
1
package eu.dnetlib.data.information.oai.publisher.conf;
2

  
3
import java.io.IOException;
4
import java.util.Collection;
5
import java.util.List;
6

  
7
import javax.annotation.Resource;
... This diff was truncated because it exceeds the maximum size that can be displayed.

Also available in: Unified diff