Project

General

Profile

« Previous | Next » 

Revision 49267

branch for solr6

View differences:

modules/dnet-index-solr-common/branches/solr6/deploy.info
1
{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-index-solr-common/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-index-solr-common"}
modules/dnet-index-solr-common/branches/solr6/src/main/java/eu/dnetlib/functionality/index/solr/feed/StreamingInputDocumentFactory.java
1
package eu.dnetlib.functionality.index.solr.feed;
2

  
3
import java.io.StringReader;
4
import java.io.StringWriter;
5
import java.util.HashMap;
6
import java.util.Iterator;
7
import java.util.List;
8
import javax.xml.stream.*;
9
import javax.xml.stream.events.Namespace;
10
import javax.xml.stream.events.StartElement;
11
import javax.xml.stream.events.XMLEvent;
12

  
13
import com.google.common.collect.Lists;
14
import eu.dnetlib.functionality.index.solr.feed.ResultTransformer.Mode;
15
import org.apache.solr.common.SolrInputDocument;
16

  
17
/**
18
 * Optimized version of the document parser, drop in replacement of InputDocumentFactory.
19
 *
20
 * <p>
21
 * Faster because:
22
 * </p>
23
 * <ul>
24
 * <li>Doesn't create a DOM for the full document</li>
25
 * <li>Doesn't execute xpaths agains the DOM</li>
26
 * <li>Quickly serialize the 'result' element directly in a string.</li>
27
 * <li>Uses less memory: less pressure on GC and allows more threads to process this in parallel</li>
28
 * </ul>
29
 *
30
 * <p>
31
 * This class is fully reentrant and can be invoked in parallel.
32
 * </p>
33
 *
34
 * @author marko
35
 *
36
 */
37
public class StreamingInputDocumentFactory extends InputDocumentFactory {
38

  
39
	protected static final String DEFAULTDNETRESULT = "dnetResult";
40

  
41
	protected static final String TARGETFIELDS = "targetFields";
42

  
43
	protected static final String INDEX_RECORD_ID_ELEMENT = "indexRecordIdentifier";
44

  
45
	protected static final String ROOT_ELEMENT = "indexRecord";
46

  
47
	protected ThreadLocal<XMLInputFactory> inputFactory = new ThreadLocal<XMLInputFactory>() {
48

  
49
		@Override
50
		protected XMLInputFactory initialValue() {
51
			return XMLInputFactory.newInstance();
52
		}
53
	};
54

  
55
	protected ThreadLocal<XMLOutputFactory> outputFactory = new ThreadLocal<XMLOutputFactory>() {
56

  
57
		@Override
58
		protected XMLOutputFactory initialValue() {
59
			return XMLOutputFactory.newInstance();
60
		}
61
	};
62

  
63
	protected ThreadLocal<XMLEventFactory> eventFactory = new ThreadLocal<XMLEventFactory>() {
64

  
65
		@Override
66
		protected XMLEventFactory initialValue() {
67
			return XMLEventFactory.newInstance();
68
		}
69
	};
70

  
71
	/**
72
	 * {@inheritDoc}
73
	 *
74
	 * @see eu.dnetlib.functionality.index.solr.feed.InputDocumentFactory#parseDocument(eu.dnetlib.functionality.index.solr.feed.IndexDocument,
75
	 * java.lang.String)
76
	 */
77
	@Override
78
	public SolrInputDocument parseDocument(final String version, final String inputDocument, final String dsId, final String resultName)
79
			throws XMLStreamException {
80
		return parseDocument(version, inputDocument, dsId, resultName, null);
81
	}
82

  
83
	/**
84
	 * {@inheritDoc}
85
	 *
86
	 * @see eu.dnetlib.functionality.index.solr.feed.InputDocumentFactory#parseDocument(eu.dnetlib.functionality.index.solr.feed.IndexDocument,
87
	 *      java.lang.String, com.google.common.base.Function)
88
	 */
89
	@Override
90
	public SolrInputDocument parseDocument(final String version,
91
			final String inputDocument,
92
			final String dsId,
93
			final String resultName,
94
			final ResultTransformer resultTransformer) {
95

  
96
		final StringWriter results = new StringWriter();
97
		final List<Namespace> nsList = Lists.newLinkedList();
98
		try {
99

  
100
			XMLEventReader parser = inputFactory.get().createXMLEventReader(new StringReader(inputDocument));
101

  
102
			final SolrInputDocument indexDocument = new SolrInputDocument(new HashMap<>());
103

  
104
			while (parser.hasNext()) {
105
				final XMLEvent event = parser.nextEvent();
106
				if ((event != null) && event.isStartElement()) {
107
					final String localName = event.asStartElement().getName().getLocalPart();
108

  
109
					if (ROOT_ELEMENT.equals(localName)) {
110
						nsList.addAll(getNamespaces(event));
111
					} else if (INDEX_RECORD_ID_ELEMENT.equals(localName)) {
112
						final XMLEvent text = parser.nextEvent();
113
						String recordId = getText(text);
114
						indexDocument.addField(INDEX_RECORD_ID, recordId);
115
					} else if (TARGETFIELDS.equals(localName)) {
116
						parseTargetFields(indexDocument, parser);
117
					} else if (resultName.equals(localName)) {
118
						if (resultTransformer == null || !(Mode.empty.equals(resultTransformer.getMode()))) {
119
							copyResult(indexDocument, results, parser, nsList, resultName, resultTransformer);
120
						}
121
					}
122
				}
123
			}
124

  
125
			if (version != null) {
126
				indexDocument.addField(DS_VERSION, version);
127
			}
128

  
129
			if (dsId != null) {
130
				indexDocument.addField(DS_ID, dsId);
131
			}
132

  
133
			if (!indexDocument.containsKey(INDEX_RECORD_ID)) {
134
				indexDocument.clear();
135
				System.err.println("missing indexrecord id:\n" + inputDocument);
136
			}
137

  
138
			return indexDocument;
139
		} catch (XMLStreamException e) {
140
			return new SolrInputDocument();
141
		}
142
	}
143

  
144
	private List<Namespace> getNamespaces(final XMLEvent event) {
145
		final List<Namespace> res = Lists.newLinkedList();
146
		@SuppressWarnings("unchecked")
147
		Iterator<Namespace> nsIter = event.asStartElement().getNamespaces();
148
		while (nsIter.hasNext()) {
149
			Namespace ns = nsIter.next();
150
			res.add(ns);
151
		}
152
		return res;
153
	}
154

  
155
	/**
156
	 * Parse the targetFields block and add fields to the solr document.
157
	 *
158
	 * @param indexDocument
159
	 * @param parser
160
	 * @throws XMLStreamException
161
	 */
162
	protected void parseTargetFields(final SolrInputDocument indexDocument, final XMLEventReader parser) throws XMLStreamException {
163

  
164
		boolean hasFields = false;
165

  
166
		while (parser.hasNext()) {
167
			final XMLEvent targetEvent = parser.nextEvent();
168
			if (targetEvent.isEndElement() && targetEvent.asEndElement().getName().getLocalPart().equals(TARGETFIELDS)) {
169
				break;
170
			}
171

  
172
			if (targetEvent.isStartElement()) {
173
				final String fieldName = targetEvent.asStartElement().getName().getLocalPart();
174
				final XMLEvent text = parser.nextEvent();
175

  
176
				String data = getText(text);
177

  
178
				addField(indexDocument, fieldName, data);
179
				hasFields = true;
180
			}
181
		}
182

  
183
		if (!hasFields) {
184
			indexDocument.clear();
185
		}
186
	}
187

  
188
	/**
189
	 * Copy the /indexRecord/result element and children, preserving namespace declarations etc.
190
	 *
191
	 * @param indexDocument
192
	 * @param results
193
	 * @param parser
194
	 * @param nsList
195
	 * @throws XMLStreamException
196
	 */
197
	protected void copyResult(final SolrInputDocument indexDocument,
198
			final StringWriter results,
199
			final XMLEventReader parser,
200
			final List<Namespace> nsList,
201
			final String dnetResult,
202
			final ResultTransformer resultTransformer) throws XMLStreamException {
203
		final XMLEventWriter writer = outputFactory.get().createXMLEventWriter(results);
204

  
205
		for (Namespace ns : nsList) {
206
			eventFactory.get().createNamespace(ns.getPrefix(), ns.getNamespaceURI());
207
		}
208

  
209
		StartElement newRecord = eventFactory.get().createStartElement("", null, RESULT, null, nsList.iterator());
210

  
211
		// new root record
212
		writer.add(newRecord);
213

  
214
		// copy the rest as it is
215
		while (parser.hasNext()) {
216
			final XMLEvent resultEvent = parser.nextEvent();
217

  
218
			// TODO: replace with depth tracking instead of close tag tracking.
219
			if (resultEvent.isEndElement() && resultEvent.asEndElement().getName().getLocalPart().equals(dnetResult)) {
220
				writer.add(eventFactory.get().createEndElement("", null, RESULT));
221
				break;
222
			}
223

  
224
			writer.add(resultEvent);
225
		}
226
		writer.close();
227

  
228
		if (resultTransformer != null) {
229
			indexDocument.addField(INDEX_RESULT, resultTransformer.apply(results.toString()));
230
		} else {
231
			indexDocument.addField(INDEX_RESULT, results.toString());
232
		}
233
	}
234

  
235
	/**
236
	 * Helper used to add a field to a solr doc. It avoids to add empy fields
237
	 *
238
	 * @param indexDocument
239
	 * @param field
240
	 * @param value
241
	 */
242
	private final void addField(final SolrInputDocument indexDocument, final String field, final String value) {
243
		String cleaned = value.trim();
244
		if (!cleaned.isEmpty()) {
245
			// log.info("\n\n adding field " + field.toLowerCase() + " value: " + cleaned + "\n");
246
			indexDocument.addField(field.toLowerCase(), cleaned);
247
		}
248
	}
249

  
250
	/**
251
	 * Helper used to get the string from a text element.
252
	 *
253
	 * @param text
254
	 * @return
255
	 */
256
	protected final String getText(final XMLEvent text) {
257
		if (text.isEndElement()) // log.warn("skipping because isEndOfElement " + text.asEndElement().getName().getLocalPart());
258
			return "";
259

  
260
		return text.asCharacters().getData();
261
	}
262

  
263
}
modules/dnet-index-solr-common/branches/solr6/src/main/java/eu/dnetlib/functionality/index/solr/feed/ResultTransformer.java
1
package eu.dnetlib.functionality.index.solr.feed;
2

  
3
import com.google.common.base.Function;
4

  
5
/**
6
 * Created by claudio on 17/11/15.
7
 */
8
public abstract class ResultTransformer implements Function<String, String> {
9

  
10
	public enum Mode {compress, empty, xslt, base64}
11

  
12
	protected Mode mode;
13

  
14
	public ResultTransformer(final Mode mode) {
15
		this.mode = mode;
16
	}
17

  
18
	public Mode getMode() {
19
		return mode;
20
	}
21

  
22
	public void setMode(final Mode mode) {
23
		this.mode = mode;
24
	}
25

  
26
}
modules/dnet-index-solr-common/branches/solr6/src/main/java/eu/dnetlib/functionality/index/solr/feed/InputDocumentFactory.java
1
package eu.dnetlib.functionality.index.solr.feed;
2

  
3
import java.text.ParseException;
4
import java.text.SimpleDateFormat;
5
import java.util.Arrays;
6
import java.util.List;
7
import javax.xml.stream.XMLStreamException;
8

  
9
import org.apache.solr.common.SolrInputDocument;
10
import org.dom4j.DocumentException;
11

  
12
/**
13
 *
14
 * @author claudio
15
 *
16
 */
17
public abstract class InputDocumentFactory {
18

  
19
	public static final String INDEX_FIELD_PREFIX = "__";
20

  
21
	public static final String DS_VERSION = INDEX_FIELD_PREFIX + "dsversion";
22

  
23
	public static final String DS_ID = INDEX_FIELD_PREFIX + "dsid";
24

  
25
	public static final String RESULT = "result";
26

  
27
	public static final String INDEX_RESULT = INDEX_FIELD_PREFIX + RESULT;
28

  
29
	public static final String INDEX_RECORD_ID = INDEX_FIELD_PREFIX + "indexrecordidentifier";
30

  
31
	private static final String outFormat = new String("yyyy-MM-dd'T'hh:mm:ss'Z'");
32

  
33
	private final static List<String> dateFormats = Arrays.asList("yyyy-MM-dd'T'hh:mm:ss", "yyyy-MM-dd", "dd-MM-yyyy", "dd/MM/yyyy", "yyyy");
34

  
35
	public abstract SolrInputDocument parseDocument(final String version,
36
			final String inputDocument,
37
			final String dsId,
38
			final String resultName) throws XMLStreamException;
39

  
40
	public abstract SolrInputDocument parseDocument(final String version,
41
			final String inputDocument,
42
			final String dsId,
43
			final String resultName,
44
			final ResultTransformer resultTransformer) throws XMLStreamException;
45

  
46
	/**
47
	 * method return a solr-compatible string representation of a date
48
	 *
49
	 * @param date
50
	 * @return
51
	 * @throws DocumentException
52
	 * @throws ParseException
53
	 */
54
	public static String getParsedDateField(final String date) {
55
		for (String formatString : dateFormats) {
56
			try {
57
				return new SimpleDateFormat(outFormat).format(new SimpleDateFormat(formatString).parse(date));
58
			} catch (ParseException e) {}
59
		}
60
		throw new IllegalStateException("unable to parse date: " + date);
61
	}
62

  
63
	public String parseDate(final String date) {
64
		return getParsedDateField(date);
65
	}
66

  
67
}
modules/dnet-index-solr-common/branches/solr6/pom.xml
1
<?xml version="1.0" encoding="UTF-8"?>
2
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
3
	<parent>
4
		<groupId>eu.dnetlib</groupId>
5
		<artifactId>dnet45-parent</artifactId>
6
		<version>1.0.0-SNAPSHOT</version>
7
		<relativePath />
8
	</parent>
9
	<modelVersion>4.0.0</modelVersion>
10
	<groupId>eu.dnetlib</groupId>
11
	<artifactId>dnet-index-solr-common</artifactId>
12
	<version>1.3.2-SNAPSHOT</version>
13
	<scm>
14
		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-index-solr-common/trunk</developerConnection>
15
	</scm>
16
	<dependencies>
17
		<dependency>
18
			<groupId>org.apache.solr</groupId>
19
			<artifactId>solr-solrj</artifactId>
20
			<version>${apache.solr.version}</version>
21
			<exclusions>
22
				<exclusion>
23
					<artifactId>wstx-asl</artifactId>
24
					<groupId>org.codehaus.woodstox</groupId>
25
				</exclusion>
26
				<exclusion>
27
					<artifactId>jcl-over-slf4j</artifactId>
28
					<groupId>org.slf4j</groupId>
29
				</exclusion>
30
			</exclusions>
31
		</dependency>
32
		<dependency>
33
			<groupId>dom4j</groupId>
34
			<artifactId>dom4j</artifactId>
35
			<version>${dom4j.version}</version>
36
		</dependency>
37
		<dependency>
38
			<groupId>com.google.guava</groupId>
39
			<artifactId>guava</artifactId>
40
			<version>${google.guava.version}</version>
41
		</dependency>
42

  
43
	</dependencies>
44
</project>

Also available in: Unified diff