Revision 53830
Added by Claudio Atzori almost 6 years ago
| modules/dnet-index-solr-service/branches/solr75/deploy.info | ||
|---|---|---|
| 1 |
{ "type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-index-solr-service/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-index-solr-service" }
|
|
| 1 |
{ "type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-index-solr-service/branches/solr75", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-index-solr-service_solr75" }
|
|
| modules/dnet-index-solr-service/branches/solr75/src/main/java/eu/dnetlib/functionality/index/solr/feed/SolrDocumentMapperFactory.java | ||
|---|---|---|
| 5 | 5 |
import eu.dnetlib.functionality.index.model.Any.ValueType; |
| 6 | 6 |
import eu.dnetlib.functionality.index.model.document.IndexDocument; |
| 7 | 7 |
import eu.dnetlib.functionality.index.model.document.Status; |
| 8 |
import eu.dnetlib.functionality.index.model.util.SolrIndexDocument;
|
|
| 8 |
import eu.dnetlib.functionality.index.query.SolrIndexDocument;
|
|
| 9 | 9 |
import eu.dnetlib.functionality.index.utils.MetadataReference; |
| 10 | 10 |
import org.springframework.beans.factory.annotation.Required; |
| 11 | 11 |
|
| ... | ... | |
| 28 | 28 |
final String dsId, |
| 29 | 29 |
final String version) {
|
| 30 | 30 |
|
| 31 |
return new Function<String, IndexDocument>() {
|
|
| 32 |
|
|
| 33 |
@Override |
|
| 34 |
public IndexDocument apply(final String doc) {
|
|
| 35 |
SolrIndexDocument indexDocument = new SolrIndexDocument(schema, dsId); |
|
| 36 |
try {
|
|
| 37 |
indexDocument.setContent(documentFactory.parseDocument(version, doc, dsId, DNETRESULT)); |
|
| 38 |
} catch (XMLStreamException e) {
|
|
| 39 |
return indexDocument.setMarked(); |
|
| 40 |
} |
|
| 41 |
indexDocument.setStatus(Status.OK); |
|
| 42 |
return indexDocument; |
|
| 31 |
return doc -> {
|
|
| 32 |
SolrIndexDocument indexDocument = new SolrIndexDocument(schema, dsId); |
|
| 33 |
try {
|
|
| 34 |
indexDocument.setContent(documentFactory.parseDocument(version, doc, dsId, DNETRESULT)); |
|
| 35 |
} catch (XMLStreamException e) {
|
|
| 36 |
return indexDocument.setMarked(); |
|
| 43 | 37 |
} |
| 38 |
indexDocument.setStatus(Status.OK); |
|
| 39 |
return indexDocument; |
|
| 44 | 40 |
}; |
| 45 | 41 |
} |
| 46 | 42 |
|
| ... | ... | |
| 48 | 44 |
public Function<String, IndexDocument> getRecordMapper(final Map<String, ValueType> schema, final MetadataReference mdRef, final String dsId, final String version, final boolean emptyResult) {
|
| 49 | 45 |
if (emptyResult == false) return getRecordMapper(schema, dsId, version); |
| 50 | 46 |
else {
|
| 51 |
return new Function<String, IndexDocument>() {
|
|
| 47 |
return doc -> {
|
|
| 48 |
SolrIndexDocument indexDocument = new SolrIndexDocument(schema, dsId); |
|
| 49 |
try {
|
|
| 52 | 50 |
|
| 53 |
@Override |
|
| 54 |
public IndexDocument apply(final String doc) {
|
|
| 55 |
SolrIndexDocument indexDocument = new SolrIndexDocument(schema, dsId); |
|
| 56 |
try {
|
|
| 57 |
|
|
| 58 |
ResultTransformer transformer = new ResultTransformer(ResultTransformer.Mode.empty) {
|
|
| 59 |
@Override |
|
| 60 |
public String apply(String input) {
|
|
| 61 |
return input; |
|
| 62 |
} |
|
| 63 |
}; |
|
| 64 |
indexDocument.setContent(documentFactory.parseDocument(version, doc, dsId, DNETRESULT, transformer)); |
|
| 65 |
} catch (XMLStreamException e) {
|
|
| 66 |
return indexDocument.setMarked(); |
|
| 67 |
} |
|
| 68 |
indexDocument.setStatus(Status.OK); |
|
| 69 |
return indexDocument; |
|
| 51 |
ResultTransformer transformer = new ResultTransformer(ResultTransformer.Mode.empty) {
|
|
| 52 |
@Override |
|
| 53 |
public String apply(String input) {
|
|
| 54 |
return input; |
|
| 55 |
} |
|
| 56 |
}; |
|
| 57 |
indexDocument.setContent(documentFactory.parseDocument(version, doc, dsId, DNETRESULT, transformer)); |
|
| 58 |
} catch (XMLStreamException e) {
|
|
| 59 |
return indexDocument.setMarked(); |
|
| 70 | 60 |
} |
| 61 |
indexDocument.setStatus(Status.OK); |
|
| 62 |
return indexDocument; |
|
| 71 | 63 |
}; |
| 72 | 64 |
} |
| 73 | 65 |
} |
| modules/dnet-index-solr-service/branches/solr75/src/main/java/eu/dnetlib/functionality/index/utils/IndexSchemaFactory.java | ||
|---|---|---|
| 42 | 42 |
} |
| 43 | 43 |
|
| 44 | 44 |
/** |
| 45 |
* @param docIn
|
|
| 45 |
* @param fields
|
|
| 46 | 46 |
* input document |
| 47 | 47 |
* @return the application of the schemaXslt transformation to the docIn document |
| 48 | 48 |
* @throws IndexServiceException |
| modules/dnet-index-solr-service/branches/solr75/src/main/java/eu/dnetlib/functionality/index/utils/IndexConfigFactory.java | ||
|---|---|---|
| 58 | 58 |
private StringTemplate solrConfig; |
| 59 | 59 |
|
| 60 | 60 |
/** |
| 61 |
* @param docIn
|
|
| 61 |
* @param params
|
|
| 62 | 62 |
* input document |
| 63 | 63 |
* @return the application of the schemaXslt transformation to the docIn |
| 64 | 64 |
* document |
| 65 | 65 |
* @throws IndexServiceException |
| 66 | 66 |
*/ |
| 67 |
public String getConfig(final Map<String, String> params) |
|
| 68 |
throws IndexServiceException {
|
|
| 67 |
public String getConfig(final Map<String, String> params) throws IndexServiceException {
|
|
| 69 | 68 |
final StringTemplate conf = new StringTemplate(getSolrConfig() |
| 70 | 69 |
.getTemplate()); |
| 71 | 70 |
for (Entry<String, String> e : filter(params).entrySet()) {
|
| ... | ... | |
| 77 | 76 |
} |
| 78 | 77 |
|
| 79 | 78 |
private Map<String, String> filter(final Map<String, String> params) {
|
| 80 |
return Maps.filterKeys(params, new Predicate<String>() {
|
|
| 81 |
|
|
| 82 |
@Override |
|
| 83 |
public boolean apply(final String input) {
|
|
| 84 |
return CONFIG_PARAMS.isValid(input); |
|
| 85 |
} |
|
| 86 |
}); |
|
| 79 |
return Maps.filterKeys(params, input -> CONFIG_PARAMS.isValid(input)); |
|
| 87 | 80 |
} |
| 88 | 81 |
|
| 89 | 82 |
public StringTemplate getSolrConfig() {
|
| modules/dnet-index-solr-service/branches/solr75/src/main/java/eu/dnetlib/functionality/index/utils/RemoteSolrAdministrator.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.functionality.index.utils; |
| 2 | 2 |
|
| 3 |
import java.io.IOException; |
|
| 4 |
import java.util.Map; |
|
| 5 |
|
|
| 6 | 3 |
import com.google.common.collect.Maps; |
| 7 | 4 |
import com.google.gson.JsonElement; |
| 8 | 5 |
import com.google.gson.JsonObject; |
| ... | ... | |
| 14 | 11 |
import org.apache.http.HttpResponse; |
| 15 | 12 |
import org.apache.http.client.HttpClient; |
| 16 | 13 |
import org.apache.http.client.methods.HttpGet; |
| 17 |
import org.apache.solr.client.solrj.SolrServer;
|
|
| 14 |
import org.apache.solr.client.solrj.SolrClient;
|
|
| 18 | 15 |
import org.apache.solr.client.solrj.SolrServerException; |
| 19 |
import org.apache.solr.client.solrj.impl.CloudSolrServer;
|
|
| 16 |
import org.apache.solr.client.solrj.impl.CloudSolrClient;
|
|
| 20 | 17 |
import org.apache.solr.client.solrj.request.LukeRequest; |
| 21 | 18 |
import org.apache.solr.client.solrj.response.LukeResponse; |
| 22 | 19 |
import org.apache.solr.client.solrj.response.LukeResponse.FieldInfo; |
| ... | ... | |
| 27 | 24 |
import org.apache.zookeeper.data.Stat; |
| 28 | 25 |
import org.springframework.beans.factory.annotation.Required; |
| 29 | 26 |
|
| 27 |
import java.io.IOException; |
|
| 28 |
import java.util.Map; |
|
| 29 |
|
|
| 30 | 30 |
public class RemoteSolrAdministrator {
|
| 31 | 31 |
|
| 32 | 32 |
/** |
| ... | ... | |
| 74 | 74 |
return false; |
| 75 | 75 |
} |
| 76 | 76 |
|
| 77 |
public boolean indexCollectionExists(final String indexCollectionId, final CloudSolrServer server) {
|
|
| 77 |
public boolean indexCollectionExists(final String indexCollectionId, final CloudSolrClient client) {
|
|
| 78 | 78 |
|
| 79 | 79 |
Watcher watcher = new Watcher() {
|
| 80 | 80 |
|
| ... | ... | |
| 83 | 83 |
}; |
| 84 | 84 |
try {
|
| 85 | 85 |
// server.connect(); |
| 86 |
SolrZkClient client = server.getZkStateReader().getZkClient();
|
|
| 87 |
if (!client.isConnected()) {
|
|
| 88 |
server.shutdown();
|
|
| 89 |
server.connect();
|
|
| 86 |
SolrZkClient zkClient = client.getZkStateReader().getZkClient();
|
|
| 87 |
if (!zkClient.isConnected()) {
|
|
| 88 |
client.close();
|
|
| 89 |
client.connect();
|
|
| 90 | 90 |
} |
| 91 | 91 |
|
| 92 |
byte[] data = client.getData("/clusterstate.json", watcher, new Stat(), true);
|
|
| 92 |
byte[] data = zkClient.getData("/clusterstate.json", watcher, new Stat(), true);
|
|
| 93 | 93 |
if (data == null) return false; |
| 94 | 94 |
String jsonLine = new String(data); |
| 95 | 95 |
JsonElement jelement = new JsonParser().parse(jsonLine); |
| 96 | 96 |
JsonObject jobject = jelement.getAsJsonObject(); |
| 97 | 97 |
if (jobject.has(indexCollectionId)) {
|
| 98 |
server.setDefaultCollection(indexCollectionId);
|
|
| 99 |
server.connect();
|
|
| 100 |
SolrPingResponse status = server.ping();
|
|
| 98 |
client.setDefaultCollection(indexCollectionId);
|
|
| 99 |
client.connect();
|
|
| 100 |
SolrPingResponse status = client.ping();
|
|
| 101 | 101 |
return (Integer) status.getResponseHeader().get("status") == 0;
|
| 102 | 102 |
} else return false; |
| 103 | 103 |
|
| ... | ... | |
| 117 | 117 |
else return ValueType.STRING; |
| 118 | 118 |
} |
| 119 | 119 |
|
| 120 |
public Map<String, ValueType> getFieldNamesAndTypes(final String coreName, final SolrServer server) throws IndexServiceException {
|
|
| 120 |
public Map<String, ValueType> getFieldNamesAndTypes(final String coreName, final SolrClient client) throws IndexServiceException {
|
|
| 121 | 121 |
try {
|
| 122 | 122 |
if (cachedSchema.get(coreName) == null) {
|
| 123 | 123 |
synchronized (cachedSchema) {
|
| 124 |
Map<String, ValueType> schema = readFieldNamesAndTypes(coreName, server);
|
|
| 124 |
Map<String, ValueType> schema = readFieldNamesAndTypes(coreName, client);
|
|
| 125 | 125 |
log.info("setting cache for schema of collection: " + coreName);
|
| 126 | 126 |
cachedSchema.put(coreName, schema); |
| 127 | 127 |
} |
| ... | ... | |
| 132 | 132 |
} |
| 133 | 133 |
} |
| 134 | 134 |
|
| 135 |
private Map<String, ValueType> readFieldNamesAndTypes(final String coreName, final SolrServer server) throws SolrServerException, IOException {
|
|
| 135 |
private Map<String, ValueType> readFieldNamesAndTypes(final String coreName, final SolrClient client) throws SolrServerException, IOException {
|
|
| 136 | 136 |
final LukeRequest request = new LukeRequest(); |
| 137 | 137 |
request.setShowSchema(true); |
| 138 | 138 |
request.setNumTerms(0); |
| 139 |
final LukeResponse response = request.process(server);
|
|
| 139 |
final LukeResponse response = request.process(client);
|
|
| 140 | 140 |
final Map<String, FieldInfo> fieldInfos = response.getFieldInfo(); |
| 141 | 141 |
final Map<String, LukeResponse.FieldTypeInfo> fieldTypeInfos = response.getFieldTypeInfo(); |
| 142 | 142 |
final Map<String, ValueType> result = Maps.newHashMap(); |
| modules/dnet-index-solr-service/branches/solr75/src/main/java/eu/dnetlib/functionality/index/SolrIndexServerDAO.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.functionality.index; |
| 2 | 2 |
|
| 3 |
import java.io.StringReader; |
|
| 4 |
import java.util.Map; |
|
| 5 |
|
|
| 3 |
import com.google.common.collect.Maps; |
|
| 4 |
import eu.dnetlib.data.provision.index.rmi.IndexServiceException; |
|
| 6 | 5 |
import eu.dnetlib.functionality.cql.CqlValueTransformerMap; |
| 6 |
import eu.dnetlib.functionality.index.feed.DocumentMapperFactory; |
|
| 7 |
import eu.dnetlib.functionality.index.model.Any.ValueType; |
|
| 8 |
import eu.dnetlib.functionality.index.query.IndexQueryFactory; |
|
| 9 |
import eu.dnetlib.functionality.index.query.SolrIndexQueryFactory; |
|
| 10 |
import eu.dnetlib.functionality.index.query.SolrIndexQueryResponseFactory; |
|
| 11 |
import eu.dnetlib.functionality.index.solr.cql.SolrTypeBasedCqlValueTransformerMapFactory; |
|
| 12 |
import eu.dnetlib.functionality.index.solr.feed.SolrDocumentMapperFactory; |
|
| 13 |
import eu.dnetlib.functionality.index.utils.*; |
|
| 7 | 14 |
import org.apache.commons.logging.Log; |
| 8 | 15 |
import org.apache.commons.logging.LogFactory; |
| 9 |
import org.apache.solr.client.solrj.impl.CloudSolrServer;
|
|
| 16 |
import org.apache.solr.client.solrj.impl.CloudSolrClient;
|
|
| 10 | 17 |
import org.dom4j.Document; |
| 11 | 18 |
import org.dom4j.DocumentException; |
| 12 | 19 |
import org.dom4j.io.SAXReader; |
| 13 | 20 |
import org.springframework.beans.factory.annotation.Autowired; |
| 14 | 21 |
import org.springframework.beans.factory.annotation.Required; |
| 15 | 22 |
|
| 16 |
import com.google.common.collect.Maps; |
|
| 23 |
import java.io.IOException; |
|
| 24 |
import java.io.StringReader; |
|
| 25 |
import java.util.Map; |
|
| 17 | 26 |
|
| 18 |
import eu.dnetlib.data.provision.index.rmi.IndexServiceException; |
|
| 19 |
|
|
| 20 |
import eu.dnetlib.functionality.index.feed.DocumentMapperFactory; |
|
| 21 |
import eu.dnetlib.functionality.index.model.Any.ValueType; |
|
| 22 |
import eu.dnetlib.functionality.index.query.IndexQueryFactory; |
|
| 23 |
import eu.dnetlib.functionality.index.query.SolrIndexQueryFactory; |
|
| 24 |
import eu.dnetlib.functionality.index.query.SolrIndexQueryResponseFactory; |
|
| 25 |
import eu.dnetlib.functionality.index.solr.cql.SolrTypeBasedCqlValueTransformerMapFactory; |
|
| 26 |
import eu.dnetlib.functionality.index.solr.feed.SolrDocumentMapperFactory; |
|
| 27 |
import eu.dnetlib.functionality.index.utils.IndexConfigFactory; |
|
| 28 |
import eu.dnetlib.functionality.index.utils.MetadataReference; |
|
| 29 |
import eu.dnetlib.functionality.index.utils.RemoteSolrAdministrator; |
|
| 30 |
import eu.dnetlib.functionality.index.utils.ZkUtils; |
|
| 31 |
|
|
| 32 | 27 |
/** |
| 33 | 28 |
* The Class SolrIndexServerDAO. |
| 34 | 29 |
*/ |
| ... | ... | |
| 69 | 64 |
*/ |
| 70 | 65 |
@Override |
| 71 | 66 |
public void createIndexCollection(final MetadataReference mdref, final String fields) throws IndexServiceException {
|
| 72 |
CloudSolrServer server = null; |
|
| 73 |
try {
|
|
| 67 |
try(CloudSolrClient client = getClient()) {
|
|
| 74 | 68 |
|
| 75 |
server = getServer(); |
|
| 76 |
server.connect(); |
|
| 69 |
client.connect(); |
|
| 77 | 70 |
|
| 78 |
if (!solrAdministrator.indexCollectionExists(mdref.toString(), server)) {
|
|
| 71 |
if (!solrAdministrator.indexCollectionExists(mdref.toString(), client)) {
|
|
| 79 | 72 |
Map<String, String> params = Maps.newHashMap(); |
| 80 | 73 |
|
| 81 | 74 |
final Map<String, String> p = getServiceProperties(); |
| ... | ... | |
| 86 | 79 |
params.put(param_Name.toString(), p.get(param_Name.toString())); |
| 87 | 80 |
} |
| 88 | 81 |
|
| 89 |
zkUtils.uploadZookeperConfig(server.getZkStateReader().getZkClient(), mdref.toString(), parse(fields), params, true);
|
|
| 82 |
zkUtils.uploadZookeperConfig(client.getZkStateReader().getZkClient(), mdref.toString(), parse(fields), params, true);
|
|
| 90 | 83 |
solrAdministrator.createSolrIndex(p.get("host"), p.get("port"), mdref.toString(), p.get("numShards"), p.get("replicationFactor"),
|
| 91 | 84 |
mdref.toString(), p.get("maxShardsPerNode"));
|
| 92 | 85 |
} |
| 93 |
server.getZkStateReader().close();
|
|
| 86 |
client.getZkStateReader().close();
|
|
| 94 | 87 |
|
| 95 | 88 |
} catch (Exception e) {
|
| 96 | 89 |
log.error("Error on creating IndexCollection", e);
|
| 97 | 90 |
throw new IndexServiceException("Error on creating IndexCollection", e);
|
| 98 |
} finally {
|
|
| 99 |
if (server != null) {
|
|
| 100 |
server.shutdown(); |
|
| 101 |
} |
|
| 102 | 91 |
} |
| 103 | 92 |
} |
| 104 | 93 |
|
| 105 | 94 |
@Override |
| 106 | 95 |
public void updateIndexCollection(final MetadataReference mdRef, final Document fields) throws IndexServiceException {
|
| 107 |
CloudSolrServer server = null; |
|
| 108 |
try {
|
|
| 109 |
server = getServer(); |
|
| 110 |
server.connect(); |
|
| 96 |
try(CloudSolrClient client = getClient()) {
|
|
| 97 |
|
|
| 98 |
client.connect(); |
|
| 111 | 99 |
Map<String, String> params = Maps.newHashMap(); |
| 112 | 100 |
|
| 113 | 101 |
params.put("numShards", getServiceProperties().get("numShards"));
|
| ... | ... | |
| 117 | 105 |
params.put(param_Name.toString(), getServiceProperties().get(param_Name.toString())); |
| 118 | 106 |
} |
| 119 | 107 |
|
| 120 |
zkUtils.uploadZookeperConfig(server.getZkStateReader().getZkClient(), mdRef.toString(), fields, params, true); |
|
| 121 |
server.getZkStateReader().close(); |
|
| 122 |
server.shutdown(); |
|
| 108 |
zkUtils.uploadZookeperConfig(client.getZkStateReader().getZkClient(), mdRef.toString(), fields, params, true); |
|
| 109 |
client.getZkStateReader().close(); |
|
| 123 | 110 |
solrAdministrator.reloadCollection(getServiceProperties().get("host"), getServiceProperties().get("port"), mdRef.toString());
|
| 124 | 111 |
|
| 125 | 112 |
} catch (Exception e) {
|
| 126 | 113 |
log.error("Error on updating IndexCollection", e);
|
| 127 | 114 |
throw new IndexServiceException("Error on updating IndexCollection", e);
|
| 128 |
} finally {
|
|
| 129 |
if (server != null) {
|
|
| 130 |
server.shutdown(); |
|
| 131 |
} |
|
| 132 | 115 |
} |
| 133 | 116 |
} |
| 134 | 117 |
|
| ... | ... | |
| 156 | 139 |
*/ |
| 157 | 140 |
@Override |
| 158 | 141 |
public IndexCollection getIndexCollection(final MetadataReference mdref) throws IndexServiceException {
|
| 159 |
CloudSolrServer newServer = getServer(mdref);
|
|
| 160 |
return new SolrIndexCollection(newServer);
|
|
| 142 |
CloudSolrClient client = getClient(mdref);
|
|
| 143 |
return new SolrIndexCollection(client);
|
|
| 161 | 144 |
} |
| 162 | 145 |
|
| 163 | 146 |
/** |
| ... | ... | |
| 167 | 150 |
*/ |
| 168 | 151 |
@Override |
| 169 | 152 |
public Map<String, ValueType> getSchema(final MetadataReference mdRef) throws IndexServiceException {
|
| 170 |
CloudSolrServer server = getServer(mdRef);
|
|
| 171 |
Map<String, ValueType> fields = solrAdministrator.getFieldNamesAndTypes(mdRef.toString(), server);
|
|
| 172 |
server.shutdown();
|
|
| 153 |
CloudSolrClient client = getClient(mdRef);
|
|
| 154 |
Map<String, ValueType> fields = solrAdministrator.getFieldNamesAndTypes(mdRef.toString(), client);
|
|
| 155 |
shutdown(mdRef);
|
|
| 173 | 156 |
return fields; |
| 174 | 157 |
} |
| 175 | 158 |
|
| ... | ... | |
| 200 | 183 |
*/ |
| 201 | 184 |
@Override |
| 202 | 185 |
public void shutdown(final MetadataReference mdRef) throws IndexServiceException {
|
| 203 |
getServer(mdRef).shutdown(); |
|
| 186 |
try {
|
|
| 187 |
getClient(mdRef).close(); |
|
| 188 |
} catch (IOException e) {
|
|
| 189 |
throw new IndexServiceException(e); |
|
| 190 |
} |
|
| 204 | 191 |
|
| 205 | 192 |
} |
| 206 | 193 |
|
| ... | ... | |
| 213 | 200 |
* @throws IndexServiceException |
| 214 | 201 |
* the index service exception |
| 215 | 202 |
*/ |
| 216 |
private CloudSolrServer getServer(final MetadataReference mdRef) throws IndexServiceException {
|
|
| 217 |
CloudSolrServer server = getServer();
|
|
| 218 |
server.setDefaultCollection(mdRef.toString());
|
|
| 219 |
return server;
|
|
| 203 |
private CloudSolrClient getClient(final MetadataReference mdRef) throws IndexServiceException {
|
|
| 204 |
CloudSolrClient client = getClient();
|
|
| 205 |
client.setDefaultCollection(mdRef.toString());
|
|
| 206 |
return client;
|
|
| 220 | 207 |
} |
| 221 | 208 |
|
| 222 | 209 |
/** |
| ... | ... | |
| 224 | 211 |
* |
| 225 | 212 |
* @return a server instance |
| 226 | 213 |
*/ |
| 227 |
private CloudSolrServer getServer() {
|
|
| 214 |
private CloudSolrClient getClient() {
|
|
| 228 | 215 |
String address = getEndpoint().get(ADDRESS); |
| 229 | 216 |
log.info("connecting to address: " + address);
|
| 230 |
return new CloudSolrServer(address); |
|
| 231 | 217 |
|
| 218 |
final ZkServers zk = ZkServers.newInstance(address); |
|
| 219 |
return new CloudSolrClient.Builder(zk.getHosts(), zk.getChroot()).build(); |
|
| 232 | 220 |
} |
| 233 | 221 |
|
| 234 | 222 |
/** |
| modules/dnet-index-solr-service/branches/solr75/src/main/java/eu/dnetlib/functionality/index/SolrIndexCollection.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.functionality.index; |
| 2 | 2 |
|
| 3 |
import java.util.Collection; |
|
| 4 |
import java.util.Iterator; |
|
| 5 |
|
|
| 3 |
import com.google.common.base.Function; |
|
| 4 |
import com.google.common.collect.Iterators; |
|
| 5 |
import com.google.common.collect.Lists; |
|
| 6 |
import eu.dnetlib.data.provision.index.rmi.IndexServiceException; |
|
| 7 |
import eu.dnetlib.functionality.index.model.document.IndexDocument; |
|
| 8 |
import eu.dnetlib.functionality.index.query.SolrIndexDocument; |
|
| 9 |
import eu.dnetlib.functionality.index.utils.IndexFieldUtility; |
|
| 6 | 10 |
import org.apache.commons.lang3.StringUtils; |
| 7 | 11 |
import org.apache.commons.logging.Log; |
| 8 | 12 |
import org.apache.commons.logging.LogFactory; |
| 9 |
import org.apache.solr.client.solrj.impl.CloudSolrServer;
|
|
| 13 |
import org.apache.solr.client.solrj.impl.CloudSolrClient;
|
|
| 10 | 14 |
import org.apache.solr.client.solrj.response.UpdateResponse; |
| 11 | 15 |
import org.apache.solr.common.SolrInputDocument; |
| 12 | 16 |
|
| 13 |
import com.google.common.base.Function;
|
|
| 14 |
import com.google.common.collect.Iterators;
|
|
| 15 |
import com.google.common.collect.Lists;
|
|
| 17 |
import java.io.IOException;
|
|
| 18 |
import java.util.Collection;
|
|
| 19 |
import java.util.Iterator;
|
|
| 16 | 20 |
|
| 17 |
import eu.dnetlib.data.provision.index.rmi.IndexServiceException; |
|
| 18 |
import eu.dnetlib.functionality.index.model.document.IndexDocument; |
|
| 19 |
import eu.dnetlib.functionality.index.model.util.SolrIndexDocument; |
|
| 20 |
import eu.dnetlib.functionality.index.utils.IndexFieldUtility; |
|
| 21 |
|
|
| 22 | 21 |
/** |
| 23 | 22 |
* The Class SolrIndexCollection. |
| 24 | 23 |
*/ |
| ... | ... | |
| 32 | 31 |
/** The Constant STATUS_INDEX_OK. */ |
| 33 | 32 |
public static final int STATUS_INDEX_OK = 0; |
| 34 | 33 |
|
| 35 |
/** The server. */
|
|
| 36 |
private CloudSolrServer server;
|
|
| 34 |
/** The client. */
|
|
| 35 |
private CloudSolrClient client;
|
|
| 37 | 36 |
|
| 38 | 37 |
private boolean shutdown = false; |
| 39 | 38 |
|
| ... | ... | |
| 41 | 40 |
* The Constructor. |
| 42 | 41 |
* |
| 43 | 42 |
* @param newServer |
| 44 |
* the server
|
|
| 43 |
* the client
|
|
| 45 | 44 |
*/ |
| 46 |
public SolrIndexCollection(final CloudSolrServer newServer) {
|
|
| 47 |
this.server = newServer;
|
|
| 48 |
server.connect();
|
|
| 45 |
public SolrIndexCollection(final CloudSolrClient newServer) {
|
|
| 46 |
this.client = newServer;
|
|
| 47 |
client.connect();
|
|
| 49 | 48 |
} |
| 50 | 49 |
|
| 51 | 50 |
/** |
| ... | ... | |
| 56 | 55 |
@Override |
| 57 | 56 |
public boolean add(final IndexDocument doc) throws IndexServiceException {
|
| 58 | 57 |
if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");
|
| 58 |
|
|
| 59 | 59 |
final SolrIndexDocument solrDocument = (SolrIndexDocument) doc; |
| 60 | 60 |
try {
|
| 61 |
final UpdateResponse response = server.add(solrDocument.getSolrDocument());
|
|
| 61 |
final UpdateResponse response = client.add(solrDocument.getSolrDocument());
|
|
| 62 | 62 |
return response.getStatus() == 0; |
| 63 | 63 |
} catch (final Exception e) {
|
| 64 | 64 |
throw new IndexServiceException("Unable to add document", e);
|
| ... | ... | |
| 83 | 83 |
}); |
| 84 | 84 |
|
| 85 | 85 |
try {
|
| 86 |
final UpdateResponse response = server.add(Lists.newArrayList(solrDocs));
|
|
| 86 |
final UpdateResponse response = client.add(Lists.newArrayList(solrDocs));
|
|
| 87 | 87 |
return response.getStatus() == 0; |
| 88 | 88 |
} catch (final Exception e) {
|
| 89 | 89 |
throw new IndexServiceException("Unable to add document", e);
|
| ... | ... | |
| 137 | 137 |
if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");
|
| 138 | 138 |
try {
|
| 139 | 139 |
log.debug("delete by query: " + query);
|
| 140 |
return server.deleteByQuery(query).getStatus() == STATUS_INDEX_OK;
|
|
| 140 |
return client.deleteByQuery(query).getStatus() == STATUS_INDEX_OK;
|
|
| 141 | 141 |
} catch (final Exception e) {
|
| 142 | 142 |
throw new IndexServiceException("unable to run delete by query: " + query, e);
|
| 143 | 143 |
} |
| ... | ... | |
| 153 | 153 |
if (isShutdown()) throw new IndexServiceException("Please get another SolrIndexCollection: this has been shut down");
|
| 154 | 154 |
try {
|
| 155 | 155 |
log.info("performing commit");
|
| 156 |
final UpdateResponse rsp = server.commit();
|
|
| 156 |
final UpdateResponse rsp = client.commit();
|
|
| 157 | 157 |
log.info(String.format("commit completed in %s, status %s", rsp.getElapsedTime(), rsp.getStatus()));
|
| 158 | 158 |
return rsp.getStatus() == STATUS_INDEX_OK; |
| 159 | 159 |
} catch (final Throwable e) {
|
| ... | ... | |
| 163 | 163 |
|
| 164 | 164 |
@Override |
| 165 | 165 |
public void shutdown() {
|
| 166 |
server.shutdown(); |
|
| 166 |
try {
|
|
| 167 |
client.close(); |
|
| 168 |
} catch (IOException e) {
|
|
| 169 |
throw new RuntimeException(e); |
|
| 170 |
} |
|
| 167 | 171 |
shutdown = true; |
| 168 | 172 |
} |
| 169 | 173 |
|
| modules/dnet-index-solr-service/branches/solr75/pom.xml | ||
|---|---|---|
| 3 | 3 |
<parent> |
| 4 | 4 |
<groupId>eu.dnetlib</groupId> |
| 5 | 5 |
<artifactId>dnet45-parent</artifactId> |
| 6 |
<version>1.0.0</version> |
|
| 6 |
<version>1.0.0-SNAPSHOT</version>
|
|
| 7 | 7 |
<relativePath /> |
| 8 | 8 |
</parent> |
| 9 | 9 |
<modelVersion>4.0.0</modelVersion> |
| 10 | 10 |
<groupId>eu.dnetlib</groupId> |
| 11 | 11 |
<artifactId>dnet-index-solr-service</artifactId> |
| 12 |
<version>2.4.6-schemaFix-SNAPSHOT</version>
|
|
| 12 |
<version>2.4.6-solr75-SNAPSHOT</version>
|
|
| 13 | 13 |
<scm> |
| 14 |
<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-index-solr-service/branches/dnet-index-solr-service-2.4.4-schemaFix</developerConnection>
|
|
| 14 |
<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-index-solr-service/branches/solr75</developerConnection>
|
|
| 15 | 15 |
</scm> |
| 16 | 16 |
<dependencies> |
| 17 | 17 |
<dependency> |
| ... | ... | |
| 21 | 21 |
</dependency> |
| 22 | 22 |
<dependency> |
| 23 | 23 |
<groupId>eu.dnetlib</groupId> |
| 24 |
<artifactId>dnet-index-solr-client</artifactId>
|
|
| 25 |
<version>[2.0.0,2.9.9)</version>
|
|
| 24 |
<artifactId>dnet-index-client</artifactId> |
|
| 25 |
<version>[2.3.2-solr75-SNAPSHOT]</version>
|
|
| 26 | 26 |
</dependency> |
| 27 | 27 |
</dependencies> |
| 28 | 28 |
</project> |
Also available in: Unified diff
updated to solr 7.5.0