Project

General

Profile

« Previous | Next » 

Revision 55808

reintegrated branch solr75 -r53828:HEAD

View differences:

modules/dnet-index-solr-service/trunk/src/main/java/eu/dnetlib/functionality/index/solr/feed/SolrDocumentMapperFactory.java
5 5
import eu.dnetlib.functionality.index.model.Any.ValueType;
6 6
import eu.dnetlib.functionality.index.model.document.IndexDocument;
7 7
import eu.dnetlib.functionality.index.model.document.Status;
8
import eu.dnetlib.functionality.index.model.util.SolrIndexDocument;
8
import eu.dnetlib.functionality.index.query.SolrIndexDocument;
9 9
import eu.dnetlib.functionality.index.utils.MetadataReference;
10 10
import org.springframework.beans.factory.annotation.Required;
11 11

  
......
28 28
                                                            final String dsId,
29 29
                                                            final String version) {
30 30

  
31
        return new Function<String, IndexDocument>() {
32

  
33
            @Override
34
            public IndexDocument apply(final String doc) {
35
                SolrIndexDocument indexDocument = new SolrIndexDocument(schema, dsId);
36
                try {
37
                    indexDocument.setContent(documentFactory.parseDocument(version, doc, dsId, DNETRESULT));
38
                } catch (XMLStreamException e) {
39
                    return indexDocument.setMarked();
40
                }
41
                indexDocument.setStatus(Status.OK);
42
                return indexDocument;
31
        return doc -> {
32
            SolrIndexDocument indexDocument = new SolrIndexDocument(schema, dsId);
33
            try {
34
                indexDocument.setContent(documentFactory.parseDocument(version, doc, dsId, DNETRESULT));
35
            } catch (XMLStreamException e) {
36
                return indexDocument.setMarked();
43 37
            }
38
            indexDocument.setStatus(Status.OK);
39
            return indexDocument;
44 40
        };
45 41
    }
46 42

  
......
48 44
    public Function<String, IndexDocument> getRecordMapper(final Map<String, ValueType> schema, final MetadataReference mdRef, final String dsId, final String version, final boolean emptyResult) {
49 45
        if (emptyResult == false) return getRecordMapper(schema, dsId, version);
50 46
        else {
51
            return new Function<String, IndexDocument>() {
47
            return doc -> {
48
                SolrIndexDocument indexDocument = new SolrIndexDocument(schema, dsId);
49
                try {
52 50

  
53
                @Override
54
                public IndexDocument apply(final String doc) {
55
                    SolrIndexDocument indexDocument = new SolrIndexDocument(schema, dsId);
56
                    try {
57

  
58
                        ResultTransformer transformer = new ResultTransformer(ResultTransformer.Mode.empty) {
59
                            @Override
60
                            public String apply(String input) {
61
                                return input;
62
                            }
63
                        };
64
                        indexDocument.setContent(documentFactory.parseDocument(version, doc, dsId, DNETRESULT, transformer));
65
                    } catch (XMLStreamException e) {
66
                        return indexDocument.setMarked();
67
                    }
68
                    indexDocument.setStatus(Status.OK);
69
                    return indexDocument;
51
                    ResultTransformer transformer = new ResultTransformer(ResultTransformer.Mode.empty) {
52
                        @Override
53
                        public String apply(String input) {
54
                            return input;
55
                        }
56
                    };
57
                    indexDocument.setContent(documentFactory.parseDocument(version, doc, dsId, DNETRESULT, transformer));
58
                } catch (XMLStreamException e) {
59
                    return indexDocument.setMarked();
70 60
                }
61
                indexDocument.setStatus(Status.OK);
62
                return indexDocument;
71 63
            };
72 64
        }
73 65
    }
modules/dnet-index-solr-service/trunk/src/main/java/eu/dnetlib/functionality/index/solr/browsing/GroupResult.java
32 32
	/**
33 33
	 * Builds a groupResult.
34 34
	 *
35
	 * @param fieldName
36
	 * @param fieldValue
35
	 * @param name
36
	 * @param value
37 37
	 * @param count
38 38
	 */
39 39
	public GroupResult(final String name, final String value, final int count) {
modules/dnet-index-solr-service/trunk/src/main/java/eu/dnetlib/functionality/index/utils/IndexSchemaFactory.java
42 42
	}
43 43

  
44 44
	/**
45
	 * @param docIn
45
	 * @param fields
46 46
	 *            input document
47 47
	 * @return the application of the schemaXslt transformation to the docIn document
48 48
	 * @throws IndexServiceException
modules/dnet-index-solr-service/trunk/src/main/java/eu/dnetlib/functionality/index/utils/IndexConfigFactory.java
46 46
	 *         document
47 47
	 * @throws IndexServiceException
48 48
	 */
49
	public String getConfig(final Map<String, String> params)
50
			throws IndexServiceException {
49
	public String getConfig(final Map<String, String> params) throws IndexServiceException {
51 50
		final StringTemplate conf = new StringTemplate(getSolrConfig()
52 51
				.getTemplate());
53 52
		for (Entry<String, String> e : filter(params).entrySet()) {
modules/dnet-index-solr-service/trunk/src/main/java/eu/dnetlib/functionality/index/utils/RemoteSolrAdministrator.java
1 1
package eu.dnetlib.functionality.index.utils;
2 2

  
3
import java.io.IOException;
4
import java.util.Map;
5

  
6 3
import com.google.common.collect.Maps;
7 4
import com.google.gson.JsonElement;
8 5
import com.google.gson.JsonObject;
......
22 19
import org.apache.solr.client.solrj.response.LukeResponse.FieldInfo;
23 20
import org.apache.solr.client.solrj.response.SolrPingResponse;
24 21
import org.apache.solr.common.cloud.SolrZkClient;
22
import org.apache.zookeeper.WatchedEvent;
25 23
import org.apache.zookeeper.Watcher;
26 24
import org.apache.zookeeper.data.Stat;
27 25
import org.springframework.beans.factory.annotation.Required;
28 26

  
27
import java.io.IOException;
28
import java.util.Map;
29

  
29 30
public class RemoteSolrAdministrator {
30 31

  
31 32
	/**
......
75 76

  
76 77
	public boolean indexCollectionExists(final String indexCollectionId, final CloudSolrClient client) {
77 78

  
78
		Watcher watcher = arg0 -> {};
79
		Watcher watcher = new Watcher() {
80

  
81
			@Override
82
			public void process(final WatchedEvent arg0) {}
83
		};
79 84
		try {
80 85
			// server.connect();
81 86
			SolrZkClient zkClient = client.getZkStateReader().getZkClient();
82 87
			if (!zkClient.isConnected()) {
88
				client.close();
83 89
				client.connect();
84 90
			}
85 91

  
modules/dnet-index-solr-service/trunk/src/main/java/eu/dnetlib/functionality/index/SolrIndexServerDAO.java
1 1
package eu.dnetlib.functionality.index;
2 2

  
3
import java.io.IOException;
4
import java.io.StringReader;
5
import java.util.Map;
6

  
3
import com.google.common.collect.Maps;
4
import eu.dnetlib.data.provision.index.rmi.IndexServiceException;
7 5
import eu.dnetlib.functionality.cql.CqlValueTransformerMap;
6
import eu.dnetlib.functionality.index.feed.DocumentMapperFactory;
7
import eu.dnetlib.functionality.index.model.Any.ValueType;
8
import eu.dnetlib.functionality.index.query.IndexQueryFactory;
9
import eu.dnetlib.functionality.index.query.SolrIndexQueryFactory;
10
import eu.dnetlib.functionality.index.query.SolrIndexQueryResponseFactory;
11
import eu.dnetlib.functionality.index.solr.cql.SolrTypeBasedCqlValueTransformerMapFactory;
12
import eu.dnetlib.functionality.index.solr.feed.SolrDocumentMapperFactory;
13
import eu.dnetlib.functionality.index.utils.*;
8 14
import org.apache.commons.logging.Log;
9 15
import org.apache.commons.logging.LogFactory;
10

  
11 16
import org.apache.solr.client.solrj.impl.CloudSolrClient;
12 17
import org.dom4j.Document;
13 18
import org.dom4j.DocumentException;
......
15 20
import org.springframework.beans.factory.annotation.Autowired;
16 21
import org.springframework.beans.factory.annotation.Required;
17 22

  
18
import com.google.common.collect.Maps;
23
import java.io.IOException;
24
import java.io.StringReader;
25
import java.util.Map;
19 26

  
20
import eu.dnetlib.data.provision.index.rmi.IndexServiceException;
21

  
22
import eu.dnetlib.functionality.index.feed.DocumentMapperFactory;
23
import eu.dnetlib.functionality.index.model.Any.ValueType;
24
import eu.dnetlib.functionality.index.query.IndexQueryFactory;
25
import eu.dnetlib.functionality.index.query.SolrIndexQueryFactory;
26
import eu.dnetlib.functionality.index.query.SolrIndexQueryResponseFactory;
27
import eu.dnetlib.functionality.index.solr.cql.SolrTypeBasedCqlValueTransformerMapFactory;
28
import eu.dnetlib.functionality.index.solr.feed.SolrDocumentMapperFactory;
29
import eu.dnetlib.functionality.index.utils.IndexConfigFactory;
30
import eu.dnetlib.functionality.index.utils.MetadataReference;
31
import eu.dnetlib.functionality.index.utils.RemoteSolrAdministrator;
32
import eu.dnetlib.functionality.index.utils.ZkUtils;
33

  
34 27
/**
35 28
 * The Class SolrIndexServerDAO.
36 29
 */
......
71 64
	 */
72 65
	@Override
73 66
	public void createIndexCollection(final MetadataReference mdref, final String fields) throws IndexServiceException {
74
		CloudSolrClient client = null;
75
		try {
76
			client = getClient();
67
		try(CloudSolrClient client = getClient()) {
68

  
77 69
			client.connect();
78 70

  
79 71
			if (!solrAdministrator.indexCollectionExists(mdref.toString(), client)) {
......
96 88
		} catch (Exception e) {
97 89
			log.error("Error on creating IndexCollection", e);
98 90
			throw new IndexServiceException("Error on creating IndexCollection", e);
99
		} finally {
100
			if (client != null) {
101
				shutdown(mdref);
102
			}
103 91
		}
104 92
	}
105 93

  
106 94
	@Override
107 95
	public void updateIndexCollection(final MetadataReference mdRef, final Document fields) throws IndexServiceException {
108
		CloudSolrClient client = null;
109
		try {
110
			client = getClient();
96
		try(CloudSolrClient client = getClient()) {
97

  
111 98
			client.connect();
112 99
			Map<String, String> params = Maps.newHashMap();
113 100

  
......
120 107

  
121 108
			zkUtils.uploadZookeperConfig(client.getZkStateReader().getZkClient(), mdRef.toString(), fields, params, true);
122 109
			client.getZkStateReader().close();
123
			shutdown(mdRef);
124 110
			solrAdministrator.reloadCollection(getServiceProperties().get("host"), getServiceProperties().get("port"), mdRef.toString());
125 111

  
126 112
		} catch (Exception e) {
127 113
			log.error("Error on updating IndexCollection", e);
128 114
			throw new IndexServiceException("Error on updating IndexCollection", e);
129
		} finally {
130
			if (client != null) {
131
				shutdown(mdRef);
132
			}
133 115
		}
134 116
	}
135 117

  
......
157 139
	 */
158 140
	@Override
159 141
	public IndexCollection getIndexCollection(final MetadataReference mdref) throws IndexServiceException {
160
		final CloudSolrClient client = getClient(mdref);
142
		CloudSolrClient client = getClient(mdref);
161 143
		return new SolrIndexCollection(client);
162 144
	}
163 145

  
......
206 188
		} catch (IOException e) {
207 189
			throw new IndexServiceException(e);
208 190
		}
191

  
209 192
	}
210 193

  
211 194
	/**
......
218 201
	 *             the index service exception
219 202
	 */
220 203
	private CloudSolrClient getClient(final MetadataReference mdRef) throws IndexServiceException {
221
		CloudSolrClient server = getClient();
222
		server.setDefaultCollection(mdRef.toString());
223
		return server;
204
		CloudSolrClient client = getClient();
205
		client.setDefaultCollection(mdRef.toString());
206
		return client;
224 207
	}
225 208

  
226 209
	/**
......
229 212
	 * @return a server instance
230 213
	 */
231 214
	private CloudSolrClient getClient() {
232
		final String address = getEndpoint().get(ADDRESS);
215
		String address = getEndpoint().get(ADDRESS);
233 216
		log.info("connecting to address: " + address);
234
		return new CloudSolrClient.Builder()
235
				.withZkHost(address)
236
				.build();
217

  
218
		final ZkServers zk = ZkServers.newInstance(address);
219
		return new CloudSolrClient.Builder(zk.getHosts(), zk.getChroot()).build();
237 220
	}
238 221

  
239 222
	/**
modules/dnet-index-solr-service/trunk/src/main/java/eu/dnetlib/functionality/index/SolrIndexCollection.java
1 1
package eu.dnetlib.functionality.index;
2 2

  
3
import java.io.IOException;
4
import java.util.Collection;
5
import java.util.Iterator;
6

  
3
import com.google.common.base.Function;
4
import com.google.common.collect.Iterators;
5
import com.google.common.collect.Lists;
6
import eu.dnetlib.data.provision.index.rmi.IndexServiceException;
7
import eu.dnetlib.functionality.index.model.document.IndexDocument;
8
import eu.dnetlib.functionality.index.query.SolrIndexDocument;
9
import eu.dnetlib.functionality.index.utils.IndexFieldUtility;
7 10
import org.apache.commons.lang3.StringUtils;
8 11
import org.apache.commons.logging.Log;
9 12
import org.apache.commons.logging.LogFactory;
......
11 14
import org.apache.solr.client.solrj.response.UpdateResponse;
12 15
import org.apache.solr.common.SolrInputDocument;
13 16

  
14
import com.google.common.base.Function;
15
import com.google.common.collect.Iterators;
16
import com.google.common.collect.Lists;
17
import java.io.IOException;
18
import java.util.Collection;
19
import java.util.Iterator;
17 20

  
18
import eu.dnetlib.data.provision.index.rmi.IndexServiceException;
19
import eu.dnetlib.functionality.index.model.document.IndexDocument;
20
import eu.dnetlib.functionality.index.model.util.SolrIndexDocument;
21
import eu.dnetlib.functionality.index.utils.IndexFieldUtility;
22

  
23 21
/**
24 22
 * The Class SolrIndexCollection.
25 23
 */
......
33 31
	/** The Constant STATUS_INDEX_OK. */
34 32
	public static final int STATUS_INDEX_OK = 0;
35 33

  
36
	/** The server. */
34
	/** The client. */
37 35
	private CloudSolrClient client;
38 36

  
39 37
	private boolean shutdown = false;
......
41 39
	/**
42 40
	 * The Constructor.
43 41
	 *
44
	 * @param client
42
	 * @param newServer
45 43
	 *            the client
46 44
	 */
47
	public SolrIndexCollection(final CloudSolrClient client) {
48
		this.client = client;
45
	public SolrIndexCollection(final CloudSolrClient newServer) {
46
		this.client = newServer;
49 47
		client.connect();
50 48
	}
51 49

  
......
57 55
	@Override
58 56
	public boolean add(final IndexDocument doc) throws IndexServiceException {
59 57
		if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");
58

  
60 59
		final SolrIndexDocument solrDocument = (SolrIndexDocument) doc;
61 60
		try {
62 61
			final UpdateResponse response = client.add(solrDocument.getSolrDocument());
modules/dnet-index-solr-service/trunk/src/main/resources/eu/dnetlib/functionality/index/conf/schemaTemplate.xslt
385 385
					<!-- Case insensitive stop word removal.
386 386
					-->
387 387
					<filter class="solr.StopFilterFactory"
388
					        ignoreCase="true"
389
					        words="stopwords.txt"
388
							ignoreCase="true"
389
							words="stopwords.txt"
390 390
					/>
391 391
					<filter class="solr.LowerCaseFilterFactory"/>
392 392
					<filter class="solr.EnglishPossessiveFilterFactory"/>
......
400 400
					<tokenizer class="solr.StandardTokenizerFactory"/>
401 401
					<filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
402 402
					<filter class="solr.StopFilterFactory"
403
					        ignoreCase="true"
404
					        words="stopwords.txt"
403
							ignoreCase="true"
404
							words="stopwords.txt"
405 405
					/>
406 406
					<filter class="solr.LowerCaseFilterFactory"/>
407 407
					<filter class="solr.EnglishPossessiveFilterFactory"/>
......
432 432
					<!-- Case insensitive stop word removal.
433 433
					-->
434 434
					<filter class="solr.StopFilterFactory"
435
					        ignoreCase="true"
436
					        words="stopwords.txt"
435
							ignoreCase="true"
436
							words="stopwords.txt"
437 437
					/>
438 438
					<filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
439 439
					<filter class="solr.LowerCaseFilterFactory"/>
......
445 445
					<tokenizer class="solr.WhitespaceTokenizerFactory"/>
446 446
					<filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
447 447
					<filter class="solr.StopFilterFactory"
448
					        ignoreCase="true"
449
					        words="stopwords.txt"
448
							ignoreCase="true"
449
							words="stopwords.txt"
450 450
					/>
451 451
					<filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
452 452
					<filter class="solr.LowerCaseFilterFactory"/>
......
496 496
					<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" />
497 497
					<filter class="solr.LowerCaseFilterFactory"/>
498 498
					<filter class="solr.ReversedWildcardFilterFactory" withOriginal="true"
499
					        maxPosAsterisk="3" maxPosQuestion="2" maxFractionAsterisk="0.33"/>
499
							maxPosAsterisk="3" maxPosQuestion="2" maxFractionAsterisk="0.33"/>
500 500
				</analyzer>
501 501
				<analyzer type="query">
502 502
					<tokenizer class="solr.StandardTokenizerFactory"/>
......
575 575
			  http://wiki.apache.org/solr/SolrAdaptersForLuceneSpatial4
576 576
			-->
577 577
			<fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
578
			           geo="true" distErrPct="0.025" maxDistErr="0.001" distanceUnits="kilometers" />
578
					   geo="true" distErrPct="0.025" maxDistErr="0.001" distanceUnits="kilometers" />
579 579

  
580 580
		</schema>
581 581
	</xsl:template>
582
</xsl:stylesheet>
582
</xsl:stylesheet>
modules/dnet-index-solr-service/trunk/src/main/resources/eu/dnetlib/functionality/index/conf/solrconfig.xml.st
1409 1409
		EditorialMarkerFactory will do exactly that:
1410 1410
	   <transformer name="qecBooster" class="org.apache.solr.response.transform.EditorialMarkerFactory" />
1411 1411
	  -->
1412
</config>
1412
</config>
modules/dnet-index-solr-service/trunk/pom.xml
3 3
	<parent>
4 4
		<groupId>eu.dnetlib</groupId>
5 5
		<artifactId>dnet45-parent</artifactId>
6
		<version>1.0.0-SNAPSHOT</version>
6
		<version>1.0.0</version>
7 7
		<relativePath />
8 8
	</parent>
9 9
	<modelVersion>4.0.0</modelVersion>
10 10
	<groupId>eu.dnetlib</groupId>
11 11
	<artifactId>dnet-index-solr-service</artifactId>
12
	<version>3.0.0-SNAPSHOT</version>
12
	<version>2.4.7-SNAPSHOT</version>
13 13
	<scm>
14 14
		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-index-solr-service/trunk</developerConnection>
15 15
	</scm>
......
17 17
		<dependency>
18 18
			<groupId>eu.dnetlib</groupId>
19 19
			<artifactId>dnet-modular-index-service</artifactId>
20
			<version>[2.0.0,2.9.9)</version>
20
			<version>[2.5.4,2.9.9)</version>
21 21
		</dependency>
22 22
		<dependency>
23 23
			<groupId>eu.dnetlib</groupId>
24
			<artifactId>dnet-index-solr-client</artifactId>
25
			<version>[2.0.0,2.9.9)</version>
26
			<exclusions>
27
				<exclusion>
28
					<groupId>org.slf4j</groupId>
29
					<artifactId>jcl-over-slf4j</artifactId>
30
				</exclusion>
31
			</exclusions>
24
			<artifactId>dnet-index-client</artifactId>
25
			<version>[2.3.4,3.0.0)</version>
32 26
		</dependency>
33 27
	</dependencies>
34 28
</project>

Also available in: Unified diff