Project

General

Profile

« Previous | Next » 

Revision 54290

[maven-release-plugin] copy for tag dnet-index-solr-common-1.3.3

View differences:

modules/dnet-index-solr-common/tags/dnet-index-solr-common-1.3.3/deploy.info
1
{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-index-solr-common/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-index-solr-common"}
modules/dnet-index-solr-common/tags/dnet-index-solr-common-1.3.3/src/main/java/eu/dnetlib/functionality/index/solr/feed/StreamingInputDocumentFactory.java
1
package eu.dnetlib.functionality.index.solr.feed;
2

  
3
import java.io.StringReader;
4
import java.io.StringWriter;
5
import java.util.HashMap;
6
import java.util.Iterator;
7
import java.util.List;
8
import javax.xml.stream.*;
9
import javax.xml.stream.events.Namespace;
10
import javax.xml.stream.events.StartElement;
11
import javax.xml.stream.events.XMLEvent;
12

  
13
import com.google.common.collect.Lists;
14
import eu.dnetlib.functionality.index.solr.feed.ResultTransformer.Mode;
15
import org.apache.solr.common.SolrInputDocument;
16

  
17
/**
18
 * Optimized version of the document parser, drop in replacement of InputDocumentFactory.
19
 *
20
 * <p>
21
 * Faster because:
22
 * </p>
23
 * <ul>
24
 * <li>Doesn't create a DOM for the full document</li>
25
 * <li>Doesn't execute xpaths agains the DOM</li>
26
 * <li>Quickly serialize the 'result' element directly in a string.</li>
27
 * <li>Uses less memory: less pressure on GC and allows more threads to process this in parallel</li>
28
 * </ul>
29
 *
30
 * <p>
31
 * This class is fully reentrant and can be invoked in parallel.
32
 * </p>
33
 *
34
 * @author marko
35
 *
36
 */
37
public class StreamingInputDocumentFactory extends InputDocumentFactory {
38

  
39
	protected static final String DEFAULTDNETRESULT = "dnetResult";
40

  
41
	protected static final String TARGETFIELDS = "targetFields";
42

  
43
	protected static final String INDEX_RECORD_ID_ELEMENT = "indexRecordIdentifier";
44

  
45
	protected static final String ROOT_ELEMENT = "indexRecord";
46

  
47
	protected static final int MAX_FIELD_LENGTH = 25000;
48

  
49
	protected ThreadLocal<XMLInputFactory> inputFactory = new ThreadLocal<XMLInputFactory>() {
50

  
51
		@Override
52
		protected XMLInputFactory initialValue() {
53
			return XMLInputFactory.newInstance();
54
		}
55
	};
56

  
57
	protected ThreadLocal<XMLOutputFactory> outputFactory = new ThreadLocal<XMLOutputFactory>() {
58

  
59
		@Override
60
		protected XMLOutputFactory initialValue() {
61
			return XMLOutputFactory.newInstance();
62
		}
63
	};
64

  
65
	protected ThreadLocal<XMLEventFactory> eventFactory = new ThreadLocal<XMLEventFactory>() {
66

  
67
		@Override
68
		protected XMLEventFactory initialValue() {
69
			return XMLEventFactory.newInstance();
70
		}
71
	};
72

  
73
	/**
74
	 * {@inheritDoc}
75
	 *
76
	 * @see eu.dnetlib.functionality.index.solr.feed.InputDocumentFactory#parseDocument(eu.dnetlib.functionality.index.solr.feed.IndexDocument,
77
	 * java.lang.String)
78
	 */
79
	@Override
80
	public SolrInputDocument parseDocument(final String version, final String inputDocument, final String dsId, final String resultName)
81
			throws XMLStreamException {
82
		return parseDocument(version, inputDocument, dsId, resultName, null);
83
	}
84

  
85
	/**
86
	 * {@inheritDoc}
87
	 *
88
	 * @see eu.dnetlib.functionality.index.solr.feed.InputDocumentFactory#parseDocument(eu.dnetlib.functionality.index.solr.feed.IndexDocument,
89
	 *      java.lang.String, com.google.common.base.Function)
90
	 */
91
	@Override
92
	public SolrInputDocument parseDocument(final String version,
93
			final String inputDocument,
94
			final String dsId,
95
			final String resultName,
96
			final ResultTransformer resultTransformer) {
97

  
98
		final StringWriter results = new StringWriter();
99
		final List<Namespace> nsList = Lists.newLinkedList();
100
		try {
101

  
102
			XMLEventReader parser = inputFactory.get().createXMLEventReader(new StringReader(inputDocument));
103

  
104
			final SolrInputDocument indexDocument = new SolrInputDocument(new HashMap<>());
105

  
106
			while (parser.hasNext()) {
107
				final XMLEvent event = parser.nextEvent();
108
				if ((event != null) && event.isStartElement()) {
109
					final String localName = event.asStartElement().getName().getLocalPart();
110

  
111
					if (ROOT_ELEMENT.equals(localName)) {
112
						nsList.addAll(getNamespaces(event));
113
					} else if (INDEX_RECORD_ID_ELEMENT.equals(localName)) {
114
						final XMLEvent text = parser.nextEvent();
115
						String recordId = getText(text);
116
						indexDocument.addField(INDEX_RECORD_ID, recordId);
117
					} else if (TARGETFIELDS.equals(localName)) {
118
						parseTargetFields(indexDocument, parser);
119
					} else if (resultName.equals(localName)) {
120
						if (resultTransformer == null || !(Mode.empty.equals(resultTransformer.getMode()))) {
121
							copyResult(indexDocument, results, parser, nsList, resultName, resultTransformer);
122
						}
123
					}
124
				}
125
			}
126

  
127
			if (version != null) {
128
				indexDocument.addField(DS_VERSION, version);
129
			}
130

  
131
			if (dsId != null) {
132
				indexDocument.addField(DS_ID, dsId);
133
			}
134

  
135
			if (!indexDocument.containsKey(INDEX_RECORD_ID)) {
136
				indexDocument.clear();
137
				System.err.println("missing indexrecord id:\n" + inputDocument);
138
			}
139

  
140
			return indexDocument;
141
		} catch (XMLStreamException e) {
142
			return new SolrInputDocument();
143
		}
144
	}
145

  
146
	private List<Namespace> getNamespaces(final XMLEvent event) {
147
		final List<Namespace> res = Lists.newLinkedList();
148
		@SuppressWarnings("unchecked")
149
		Iterator<Namespace> nsIter = event.asStartElement().getNamespaces();
150
		while (nsIter.hasNext()) {
151
			Namespace ns = nsIter.next();
152
			res.add(ns);
153
		}
154
		return res;
155
	}
156

  
157
	/**
158
	 * Parse the targetFields block and add fields to the solr document.
159
	 *
160
	 * @param indexDocument
161
	 * @param parser
162
	 * @throws XMLStreamException
163
	 */
164
	protected void parseTargetFields(final SolrInputDocument indexDocument, final XMLEventReader parser) throws XMLStreamException {
165

  
166
		boolean hasFields = false;
167

  
168
		while (parser.hasNext()) {
169
			final XMLEvent targetEvent = parser.nextEvent();
170
			if (targetEvent.isEndElement() && targetEvent.asEndElement().getName().getLocalPart().equals(TARGETFIELDS)) {
171
				break;
172
			}
173

  
174
			if (targetEvent.isStartElement()) {
175
				final String fieldName = targetEvent.asStartElement().getName().getLocalPart();
176
				final XMLEvent text = parser.nextEvent();
177

  
178
				String data = getText(text);
179

  
180
				addField(indexDocument, fieldName, data);
181
				hasFields = true;
182
			}
183
		}
184

  
185
		if (!hasFields) {
186
			indexDocument.clear();
187
		}
188
	}
189

  
190
	/**
191
	 * Copy the /indexRecord/result element and children, preserving namespace declarations etc.
192
	 *
193
	 * @param indexDocument
194
	 * @param results
195
	 * @param parser
196
	 * @param nsList
197
	 * @throws XMLStreamException
198
	 */
199
	protected void copyResult(final SolrInputDocument indexDocument,
200
			final StringWriter results,
201
			final XMLEventReader parser,
202
			final List<Namespace> nsList,
203
			final String dnetResult,
204
			final ResultTransformer resultTransformer) throws XMLStreamException {
205
		final XMLEventWriter writer = outputFactory.get().createXMLEventWriter(results);
206

  
207
		for (Namespace ns : nsList) {
208
			eventFactory.get().createNamespace(ns.getPrefix(), ns.getNamespaceURI());
209
		}
210

  
211
		StartElement newRecord = eventFactory.get().createStartElement("", null, RESULT, null, nsList.iterator());
212

  
213
		// new root record
214
		writer.add(newRecord);
215

  
216
		// copy the rest as it is
217
		while (parser.hasNext()) {
218
			final XMLEvent resultEvent = parser.nextEvent();
219

  
220
			// TODO: replace with depth tracking instead of close tag tracking.
221
			if (resultEvent.isEndElement() && resultEvent.asEndElement().getName().getLocalPart().equals(dnetResult)) {
222
				writer.add(eventFactory.get().createEndElement("", null, RESULT));
223
				break;
224
			}
225

  
226
			writer.add(resultEvent);
227
		}
228
		writer.close();
229

  
230
		if (resultTransformer != null) {
231
			indexDocument.addField(INDEX_RESULT, resultTransformer.apply(results.toString()));
232
		} else {
233
			indexDocument.addField(INDEX_RESULT, results.toString());
234
		}
235
	}
236

  
237
	/**
238
	 * Helper used to add a field to a solr doc. It avoids to add empy fields
239
	 *
240
	 * @param indexDocument
241
	 * @param field
242
	 * @param value
243
	 */
244
	private final void addField(final SolrInputDocument indexDocument, final String field, final String value) {
245
		String cleaned = value.trim();
246
		if (!cleaned.isEmpty()) {
247
			// log.info("\n\n adding field " + field.toLowerCase() + " value: " + cleaned + "\n");
248
			indexDocument.addField(field.toLowerCase(), cleaned);
249
		}
250
	}
251

  
252
	/**
253
	 * Helper used to get the string from a text element.
254
	 *
255
	 * @param text
256
	 * @return
257
	 */
258
	protected final String getText(final XMLEvent text) {
259
		if (text.isEndElement()) // log.warn("skipping because isEndOfElement " + text.asEndElement().getName().getLocalPart());
260
			return "";
261

  
262
		final String data = text.asCharacters().getData();
263
		if (data != null && data.length() > MAX_FIELD_LENGTH) {
264
			return data.substring(0, MAX_FIELD_LENGTH);
265
		}
266

  
267
		return data;
268
	}
269

  
270
}
modules/dnet-index-solr-common/tags/dnet-index-solr-common-1.3.3/src/main/java/eu/dnetlib/functionality/index/solr/feed/ResultTransformer.java
1
package eu.dnetlib.functionality.index.solr.feed;
2

  
3
import com.google.common.base.Function;
4

  
5
/**
6
 * Created by claudio on 17/11/15.
7
 */
8
public abstract class ResultTransformer implements Function<String, String> {
9

  
10
	public enum Mode {compress, empty, xslt, base64}
11

  
12
	protected Mode mode;
13

  
14
	public ResultTransformer(final Mode mode) {
15
		this.mode = mode;
16
	}
17

  
18
	public Mode getMode() {
19
		return mode;
20
	}
21

  
22
	public void setMode(final Mode mode) {
23
		this.mode = mode;
24
	}
25

  
26
}
modules/dnet-index-solr-common/tags/dnet-index-solr-common-1.3.3/src/main/java/eu/dnetlib/functionality/index/solr/feed/InputDocumentFactory.java
1
package eu.dnetlib.functionality.index.solr.feed;
2

  
3
import java.text.ParseException;
4
import java.text.SimpleDateFormat;
5
import java.util.Arrays;
6
import java.util.List;
7
import javax.xml.stream.XMLStreamException;
8

  
9
import org.apache.solr.common.SolrInputDocument;
10
import org.dom4j.DocumentException;
11

  
12
/**
13
 *
14
 * @author claudio
15
 *
16
 */
17
public abstract class InputDocumentFactory {
18

  
19
	public static final String INDEX_FIELD_PREFIX = "__";
20

  
21
	public static final String DS_VERSION = INDEX_FIELD_PREFIX + "dsversion";
22

  
23
	public static final String DS_ID = INDEX_FIELD_PREFIX + "dsid";
24

  
25
	public static final String RESULT = "result";
26

  
27
	public static final String INDEX_RESULT = INDEX_FIELD_PREFIX + RESULT;
28

  
29
	public static final String INDEX_RECORD_ID = INDEX_FIELD_PREFIX + "indexrecordidentifier";
30

  
31
	private static final String outFormat = new String("yyyy-MM-dd'T'hh:mm:ss'Z'");
32

  
33
	private final static List<String> dateFormats = Arrays.asList("yyyy-MM-dd'T'hh:mm:ss", "yyyy-MM-dd", "dd-MM-yyyy", "dd/MM/yyyy", "yyyy");
34

  
35
	public abstract SolrInputDocument parseDocument(final String version,
36
			final String inputDocument,
37
			final String dsId,
38
			final String resultName) throws XMLStreamException;
39

  
40
	public abstract SolrInputDocument parseDocument(final String version,
41
			final String inputDocument,
42
			final String dsId,
43
			final String resultName,
44
			final ResultTransformer resultTransformer) throws XMLStreamException;
45

  
46
	/**
47
	 * method return a solr-compatible string representation of a date
48
	 *
49
	 * @param date
50
	 * @return
51
	 * @throws DocumentException
52
	 * @throws ParseException
53
	 */
54
	public static String getParsedDateField(final String date) {
55
		for (String formatString : dateFormats) {
56
			try {
57
				return new SimpleDateFormat(outFormat).format(new SimpleDateFormat(formatString).parse(date));
58
			} catch (ParseException e) {}
59
		}
60
		throw new IllegalStateException("unable to parse date: " + date);
61
	}
62

  
63
	public String parseDate(final String date) {
64
		return getParsedDateField(date);
65
	}
66

  
67
}
modules/dnet-index-solr-common/tags/dnet-index-solr-common-1.3.3/pom.xml
1
<?xml version="1.0" encoding="UTF-8"?>
2
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
3
	<parent>
4
		<groupId>eu.dnetlib</groupId>
5
		<artifactId>dnet45-parent</artifactId>
6
		<version>1.0.0</version>
7
		<relativePath />
8
	</parent>
9
	<modelVersion>4.0.0</modelVersion>
10
	<groupId>eu.dnetlib</groupId>
11
	<artifactId>dnet-index-solr-common</artifactId>
12
	<version>1.3.3</version>
13
	<scm>
14
		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-index-solr-common/tags/dnet-index-solr-common-1.3.3</developerConnection>
15
	</scm>
16
	<dependencies>
17
		<dependency>
18
			<groupId>org.apache.solr</groupId>
19
			<artifactId>solr-solrj</artifactId>
20
			<version>${apache.solr.version}</version>
21
			<exclusions>
22
				<exclusion>
23
					<artifactId>wstx-asl</artifactId>
24
					<groupId>org.codehaus.woodstox</groupId>
25
				</exclusion>
26
				<exclusion>
27
					<artifactId>jcl-over-slf4j</artifactId>
28
					<groupId>org.slf4j</groupId>
29
				</exclusion>
30
			</exclusions>
31
		</dependency>
32
		<dependency>
33
			<groupId>dom4j</groupId>
34
			<artifactId>dom4j</artifactId>
35
			<version>${dom4j.version}</version>
36
		</dependency>
37
		<dependency>
38
			<groupId>com.google.guava</groupId>
39
			<artifactId>guava</artifactId>
40
			<version>${google.guava.version}</version>
41
		</dependency>
42

  
43
	</dependencies>
44
</project>

Also available in: Unified diff