Project

General

Profile

« Previous | Next » 

Revision 42149

Added by Sandro La Bruzzo over 8 years ago

update the way to query the solr index if some collection exists

View differences:

modules/dnet-data-provision-services/trunk/src/main/java/eu/dnetlib/index/solr/SolrIndexServerDAO.java
84 84 

  		




  85
  85
  
    
			client = getClient();


  		




  86
  86
  
    
			client.connect();


  		




  
  87
  
    
			final Map<String, String> p = getServiceProperties();


  		




  87
  88
  
    

  		




  88
  
  
    
			if (!solrAdministrator.indexCollectionExists(mdref.toString(), client)) {


  		




  
  89
  
    
			if (!solrAdministrator.indexCollectionExists(mdref.toString(), client, p.get("host"), p.get("port"))) {


  		




  89
  90
  
    
				Map<String, String> params = new HashMap<>();


  		




  90
  91
  
    

  		




  91
  
  
    
				final Map<String, String> p = getServiceProperties();


  		




  92
  92
  
    
				params.put("numShards", p.get("numShards"));


  		




  93
  93
  
    
				params.put("replicationFactor", p.get("replicationFactor"));


  		




  94
  94
  
    

  		












  

    
      modules/dnet-data-provision-services/trunk/src/main/java/eu/dnetlib/index/utils/RemoteSolrAdministrator.java
    
  





  1
  1
  
    
package eu.dnetlib.index.utils;


  		




  2
  2
  
    

  		




  3
  3
  
    
import java.io.IOException;


  		




  
  4
  
    
import java.net.URL;


  		




  4
  5
  
    
import java.util.HashMap;


  		




  5
  6
  
    
import java.util.Map;


  		




  6
  7
  
    

  		




  
  8
  
    
import com.google.gson.JsonArray;


  		




  7
  9
  
    
import com.google.gson.JsonElement;


  		




  8
  10
  
    
import com.google.gson.JsonObject;


  		




  9
  11
  
    
import com.google.gson.JsonParser;


  		




  10
  12
  
    
import eu.dnetlib.clients.index.model.Any.ValueType;


  		




  11
  13
  
    
import eu.dnetlib.rmi.provision.IndexServiceException;


  		




  
  14
  
    
import org.apache.commons.io.IOUtils;


  		




  12
  15
  
    
import org.apache.commons.logging.Log;


  		




  13
  16
  
    
import org.apache.commons.logging.LogFactory;


  		




  14
  17
  
    
import org.apache.http.HttpResponse;


  		




  ......




  21
  24
  
    
import org.apache.solr.client.solrj.response.LukeResponse;


  		




  22
  25
  
    
import org.apache.solr.client.solrj.response.LukeResponse.FieldInfo;


  		




  23
  26
  
    
import org.apache.solr.client.solrj.response.SolrPingResponse;


  		




  24
  
  
    
import org.apache.solr.common.cloud.SolrZkClient;


  		




  25
  
  
    
import org.apache.zookeeper.data.Stat;


  		




  26
  27
  
    
import org.springframework.beans.factory.annotation.Required;


  		




  27
  28
  
    

  		




  28
  29
  
    
public class RemoteSolrAdministrator {


  		




  ......




  42
  43
  
    
	 * The create url request.


  		




  43
  44
  
    
	 */


  		




  44
  45
  
    
	private static String reloadURLRequest = "http://%s:%s/solr/admin/collections?action=RELOAD&name=%s";


  		




  
  46
  
    

  		




  
  47
  
    
	private static String listCollectionUrl = "http://%s:%s/solr/admin/collections?action=LIST&wt=json";


  		




  
  48
  
    

  		




  45
  49
  
    
	protected Map<String, Map<String, ValueType>> cachedSchema;


  		




  46
  50
  
    
	/**


  		




  47
  51
  
    
	 * The http client.


  		




  ......




  76
  80
  
    
		return false;


  		




  77
  81
  
    
	}


  		




  78
  82
  
    

  		




  79
  
  
    
	public boolean indexCollectionExists(final String indexCollectionId, final CloudSolrClient indexClient) {


  		




  
  83
  
    
	public boolean indexCollectionExists(final String indexCollectionId, final CloudSolrClient indexClient, final String host, final String port) {


  		




  80
  84
  
    
		try {


  		




  81
  
  
    
			// server.connect();


  		




  82
  
  
    
			SolrZkClient client = indexClient.getZkStateReader().getZkClient();


  		




  83
  
  
    
			if (!client.isConnected()) {


  		




  84
  
  
    
				indexClient.shutdown();


  		




  85
  
  
    
				indexClient.connect();


  		




  
  85
  
    

  		




  
  86
  
    
			URL url = new URL(String.format(listCollectionUrl, host, port));


  		




  
  87
  
    

  		




  
  88
  
    
			final String collectionJson = IOUtils.toString(url.openStream());


  		




  
  89
  
    
			final JsonElement jelement = new JsonParser().parse(collectionJson);


  		




  
  90
  
    
			final JsonObject jobject = jelement.getAsJsonObject();


  		




  
  91
  
    
			final JsonArray collections = jobject.getAsJsonArray("collections");


  		




  
  92
  
    

  		




  
  93
  
    
			for (JsonElement element : collections) {


  		




  
  94
  
    

  		




  
  95
  
    
				final String collectionName = element.getAsString();


  		




  
  96
  
    
				if (collectionName.contains(indexCollectionId)) {


  		




  
  97
  
    
					indexClient.setDefaultCollection(indexCollectionId);


  		




  
  98
  
    
					indexClient.connect();


  		




  
  99
  
    
					SolrPingResponse status = indexClient.ping();


  		




  
  100
  
    
					return (Integer) status.getResponseHeader().get("status") == 0;


  		




  
  101
  
    
				}


  		




  86
  102
  
    
			}


  		




  87
  103
  
    

  		




  88
  
  
    
			byte[] data = client.getData("/clusterstate.json", it -> {


  		




  89
  
  
    
			}, new Stat(), true);


  		




  90
  
  
    
			if (data == null) return false;


  		




  91
  
  
    
			String jsonLine = new String(data);


  		




  92
  
  
    
			JsonElement jelement = new JsonParser().parse(jsonLine);


  		




  93
  
  
    
			JsonObject jobject = jelement.getAsJsonObject();


  		




  94
  
  
    
			if (jobject.has(indexCollectionId)) {


  		




  95
  
  
    
				indexClient.setDefaultCollection(indexCollectionId);


  		




  96
  
  
    
				indexClient.connect();


  		




  97
  
  
    
				SolrPingResponse status = indexClient.ping();


  		




  98
  
  
    
				return (Integer) status.getResponseHeader().get("status") == 0;


  		




  99
  
  
    
			} else return false;


  		




  
  104
  
    
			return false;


  		




  100
  105
  
    

  		




  
  106
  
    
			//			DnetStreamSupport.generateStreamFromIterator(collections.iterator()).filter(it -> it.getAsString())


  		




  
  107
  
    
			//


  		




  
  108
  
    
			//


  		




  
  109
  
    
			//


  		




  
  110
  
    
			//			byte[] data = client.getData("/clusterstate.json", it -> {


  		




  
  111
  
    
			//			}, new Stat(), true);


  		




  
  112
  
    
			//			if (data == null) return false;


  		




  
  113
  
    
			//			String jsonLine = new String(data);


  		




  
  114
  
    
			//			JsonElement jelement = new JsonParser().parse(jsonLine);


  		




  
  115
  
    
			//			JsonObject jobject = jelement.getAsJsonObject();


  		




  
  116
  
    
			//			if (jobject.has(indexCollectionId)) {


  		




  
  117
  
    
			//				indexClient.setDefaultCollection(indexCollectionId);


  		




  
  118
  
    
			//				indexClient.connect();


  		




  
  119
  
    
			//				SolrPingResponse status = indexClient.ping();


  		




  
  120
  
    
			//				return (Integer) status.getResponseHeader().get("status") == 0;


  		




  
  121
  
    
			//			} else return false;


  		




  
  122
  
    

  		




  101
  123
  
    
		} catch (Exception e) {


  		




  102
  124
  
    
			log.error(e);


  		




  103
  125
  
    
			return false;


  		












  

    
      modules/dnet-data-provision-services/trunk/pom.xml
    
  





  69
  69
  
    

  		




  70
  70
  
    

  		




  71
  71
  
    
		<dependency>


  		




  
  72
  
    
			<groupId>commons-io</groupId>


  		




  
  73
  
    
			<artifactId>commons-io</artifactId>


  		




  
  74
  
    
			<version>${commons.io.version}</version>


  		




  
  75
  
    
		</dependency>


  		




  
  76
  
    

  		




  
  77
  
    

  		




  
  78
  
    
		<dependency>


  		




  72
  79
  
    
			<groupId>org.apache.solr</groupId>


  		




  73
  80
  
    
			<artifactId>solr-solrj</artifactId>


  		




  74
  81
  
    
			<version>${apache.solr.version}</version>


  		












Also available in:  Unified diff