Revision 57157
Added by Enrico Ottonello over 4 years ago
| RemoteSolrAdministrator.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.index.utils; |
| 2 | 2 |
|
| 3 | 3 |
import java.io.IOException; |
| 4 |
import java.net.URL; |
|
| 5 |
import java.util.HashMap; |
|
| 6 | 4 |
import java.util.Map; |
| 7 | 5 |
|
| 8 |
import com.google.gson.JsonArray; |
|
| 9 |
import com.google.gson.JsonElement; |
|
| 10 |
import com.google.gson.JsonObject; |
|
| 11 |
import com.google.gson.JsonParser; |
|
| 12 |
import eu.dnetlib.clients.index.model.Any.ValueType; |
|
| 13 |
import eu.dnetlib.rmi.provision.IndexServiceException; |
|
| 14 |
import org.apache.commons.io.IOUtils; |
|
| 15 | 6 |
import org.apache.commons.logging.Log; |
| 16 | 7 |
import org.apache.commons.logging.LogFactory; |
| 17 | 8 |
import org.apache.http.HttpResponse; |
| ... | ... | |
| 24 | 15 |
import org.apache.solr.client.solrj.response.LukeResponse; |
| 25 | 16 |
import org.apache.solr.client.solrj.response.LukeResponse.FieldInfo; |
| 26 | 17 |
import org.apache.solr.client.solrj.response.SolrPingResponse; |
| 18 |
import org.apache.solr.common.cloud.ClusterState; |
|
| 19 |
import org.apache.solr.common.cloud.DocCollection; |
|
| 20 |
import org.apache.solr.common.cloud.Replica; |
|
| 21 |
import org.apache.solr.common.cloud.SolrZkClient; |
|
| 22 |
import org.apache.zookeeper.WatchedEvent; |
|
| 23 |
import org.apache.zookeeper.Watcher; |
|
| 24 |
import org.apache.zookeeper.data.Stat; |
|
| 27 | 25 |
import org.springframework.beans.factory.annotation.Required; |
| 28 | 26 |
|
| 27 |
import com.google.common.collect.Maps; |
|
| 28 |
import com.google.gson.JsonElement; |
|
| 29 |
import com.google.gson.JsonObject; |
|
| 30 |
import com.google.gson.JsonParser; |
|
| 31 |
|
|
| 32 |
import eu.dnetlib.clients.index.model.Any.ValueType; |
|
| 33 |
import eu.dnetlib.rmi.provision.IndexServiceException; |
|
| 34 |
|
|
| 29 | 35 |
public class RemoteSolrAdministrator {
|
| 30 | 36 |
|
| 31 | 37 |
/** |
| ... | ... | |
| 33 | 39 |
*/ |
| 34 | 40 |
private static final Log log = LogFactory.getLog(RemoteSolrAdministrator.class); |
| 35 | 41 |
|
| 36 |
/** |
|
| 37 |
* The create url request. |
|
| 38 |
*/ |
|
| 39 |
private static String createURLRequest = |
|
| 40 |
"http://%s:%s/solr/admin/collections?action=CREATE&name=%s&numShards=%s&replicationFactor=%s&collection.configName=%s"; |
|
| 42 |
/** The create url request. */ |
|
| 43 |
private static String createURLRequest = "http://%s:%s/solr/admin/collections?action=CREATE&name=%s&numShards=%s&replicationFactor=%s&maxShardsPerNode=%s&collection.configName=%s"; |
|
| 41 | 44 |
|
| 42 |
/** |
|
| 43 |
* The create url request. |
|
| 44 |
*/ |
|
| 45 |
/** The create url request. */ |
|
| 45 | 46 |
private static String reloadURLRequest = "http://%s:%s/solr/admin/collections?action=RELOAD&name=%s"; |
| 46 | 47 |
|
| 47 |
private static String listCollectionUrl = "http://%s:%s/solr/admin/collections?action=LIST&wt=json"; |
|
| 48 |
/** The http client. */ |
|
| 49 |
private HttpClient httpClient; |
|
| 48 | 50 |
|
| 49 | 51 |
protected Map<String, Map<String, ValueType>> cachedSchema; |
| 50 |
/** |
|
| 51 |
* The http client. |
|
| 52 |
*/ |
|
| 53 |
private HttpClient httpClient; |
|
| 54 | 52 |
|
| 55 | 53 |
public RemoteSolrAdministrator() {
|
| 56 |
this.cachedSchema = new HashMap<>();
|
|
| 54 |
this.cachedSchema = Maps.newHashMap();
|
|
| 57 | 55 |
} |
| 58 | 56 |
|
| 59 | 57 |
public boolean createSolrIndex(final String host, |
| ... | ... | |
| 61 | 59 |
final String collectionName, |
| 62 | 60 |
final String numShard, |
| 63 | 61 |
final String replicationFactor, |
| 64 |
final String collectionConfigName) throws IndexServiceException {
|
|
| 62 |
final String maxShardsPerNode, |
|
| 63 |
final String collectionConfigName) throws IndexServiceException {
|
|
| 65 | 64 |
|
| 66 |
final String uri = generateCreateIndexRequest(host, port, collectionName, numShard, replicationFactor, collectionConfigName); |
|
| 65 |
final String uri = generateCreateIndexRequest(host, port, collectionName, numShard, replicationFactor, maxShardsPerNode, collectionConfigName);
|
|
| 67 | 66 |
log.info(uri); |
| 68 | 67 |
HttpGet request = new HttpGet(uri); |
| 69 | 68 |
HttpResponse response; |
| ... | ... | |
| 80 | 79 |
return false; |
| 81 | 80 |
} |
| 82 | 81 |
|
| 83 |
public boolean indexCollectionExists(final String indexCollectionId, final CloudSolrClient indexClient, final String host, final String port) {
|
|
| 82 |
public boolean indexCollectionExists(final String indexCollectionId, final CloudSolrClient client) {
|
|
| 83 |
log.debug("indexCollectionExists " + indexCollectionId + " " + client.toString());
|
|
| 84 |
Watcher watcher = new Watcher() {
|
|
| 85 |
|
|
| 86 |
@Override |
|
| 87 |
public void process(final WatchedEvent arg0) {}
|
|
| 88 |
}; |
|
| 84 | 89 |
try {
|
| 90 |
// server.connect(); |
|
| 91 |
SolrZkClient zkClient = client.getZkStateReader().getZkClient(); |
|
| 92 |
if (!zkClient.isConnected()) {
|
|
| 93 |
client.close(); |
|
| 94 |
client.connect(); |
|
| 95 |
} |
|
| 85 | 96 |
|
| 86 |
URL url = new URL(String.format(listCollectionUrl, host, port)); |
|
| 87 |
|
|
| 88 |
final String collectionJson = IOUtils.toString(url.openStream()); |
|
| 89 |
final JsonElement jelement = new JsonParser().parse(collectionJson); |
|
| 90 |
final JsonObject jobject = jelement.getAsJsonObject(); |
|
| 91 |
final JsonArray collections = jobject.getAsJsonArray("collections");
|
|
| 92 |
|
|
| 93 |
for (JsonElement element : collections) {
|
|
| 94 |
|
|
| 95 |
final String collectionName = element.getAsString(); |
|
| 96 |
if (collectionName.contains(indexCollectionId)) {
|
|
| 97 |
indexClient.setDefaultCollection(indexCollectionId); |
|
| 98 |
indexClient.connect(); |
|
| 99 |
SolrPingResponse status = indexClient.ping(); |
|
| 100 |
return (Integer) status.getResponseHeader().get("status") == 0;
|
|
| 101 |
} |
|
| 97 |
final ClusterState clusterState = client.getZkStateReader().getClusterState(); |
|
| 98 |
log.debug("indexCollectionExists clusterState " + clusterState);
|
|
| 99 |
final DocCollection collection = clusterState.getCollectionOrNull(indexCollectionId); |
|
| 100 |
log.debug("indexCollectionExists collection " + collection);
|
|
| 101 |
if (collection!=null) {
|
|
| 102 |
client.setDefaultCollection(indexCollectionId); |
|
| 103 |
client.connect(); |
|
| 104 |
SolrPingResponse status = client.ping(); |
|
| 105 |
log.debug("indexCollectionExists status.getResponseHeader() " + status.getResponseHeader());
|
|
| 106 |
return (Integer) status.getResponseHeader().get("status") == 0;
|
|
| 107 |
} else {
|
|
| 108 |
log.debug("indexCollectionExists jobject.has(indexCollectionId) FALSE " );
|
|
| 109 |
return false; |
|
| 102 | 110 |
} |
| 103 | 111 |
|
| 104 |
return false; |
|
| 105 |
|
|
| 106 |
// DnetStreamSupport.generateStreamFromIterator(collections.iterator()).filter(it -> it.getAsString()) |
|
| 107 |
// |
|
| 108 |
// |
|
| 109 |
// |
|
| 110 |
// byte[] data = client.getData("/clusterstate.json", it -> {
|
|
| 111 |
// }, new Stat(), true); |
|
| 112 |
// if (data == null) return false; |
|
| 113 |
// String jsonLine = new String(data); |
|
| 114 |
// JsonElement jelement = new JsonParser().parse(jsonLine); |
|
| 115 |
// JsonObject jobject = jelement.getAsJsonObject(); |
|
| 116 |
// if (jobject.has(indexCollectionId)) {
|
|
| 117 |
// indexClient.setDefaultCollection(indexCollectionId); |
|
| 118 |
// indexClient.connect(); |
|
| 119 |
// SolrPingResponse status = indexClient.ping(); |
|
| 120 |
// return (Integer) status.getResponseHeader().get("status") == 0;
|
|
| 121 |
// } else return false; |
|
| 122 |
|
|
| 123 | 112 |
} catch (Exception e) {
|
| 124 | 113 |
log.error(e); |
| 125 | 114 |
return false; |
| ... | ... | |
| 158 | 147 |
final LukeResponse response = request.process(client); |
| 159 | 148 |
final Map<String, FieldInfo> fieldInfos = response.getFieldInfo(); |
| 160 | 149 |
final Map<String, LukeResponse.FieldTypeInfo> fieldTypeInfos = response.getFieldTypeInfo(); |
| 161 |
final Map<String, ValueType> result = new HashMap<>();
|
|
| 150 |
final Map<String, ValueType> result = Maps.newHashMap();
|
|
| 162 | 151 |
for (FieldInfo fieldInfo : fieldInfos.values()) {
|
| 163 | 152 |
LukeResponse.FieldTypeInfo fieldTypeInfo = fieldTypeInfos.get(fieldInfo.getType()); |
| 164 | 153 |
final String fieldName = fieldTypeInfo.getName().toLowerCase(); |
| ... | ... | |
| 173 | 162 |
final String collectionName, |
| 174 | 163 |
final String numShard, |
| 175 | 164 |
final String replicationFactor, |
| 176 |
final String collectionConfigName) {
|
|
| 177 |
return String.format(createURLRequest, host, port, collectionName, numShard, replicationFactor, collectionConfigName); |
|
| 165 |
final String collectionConfigName, |
|
| 166 |
final String maxShardsPerNode) {
|
|
| 167 |
return String.format(createURLRequest, host, port, collectionName, numShard, replicationFactor, maxShardsPerNode, collectionConfigName); |
|
| 178 | 168 |
} |
| 179 | 169 |
|
| 180 | 170 |
private String generateUpdateIndexRequest(final String host, final String port, final String collectionName) {
|
| ... | ... | |
| 189 | 179 |
} |
| 190 | 180 |
|
| 191 | 181 |
/** |
| 192 |
* @param httpClient the httpClient to set |
|
| 182 |
* @param httpClient |
|
| 183 |
* the httpClient to set |
|
| 193 | 184 |
*/ |
| 194 | 185 |
@Required |
| 195 | 186 |
public void setHttpClient(final HttpClient httpClient) {
|
Also available in: Unified diff
solr 772 integration