Revision 57157
Added by Enrico Ottonello over 4 years ago
| SolrIndexServerDAO.java | ||
|---|---|---|
| 2 | 2 |
|
| 3 | 3 |
import java.io.IOException; |
| 4 | 4 |
import java.io.StringReader; |
| 5 |
import java.util.HashMap; |
|
| 6 | 5 |
import java.util.Map; |
| 7 | 6 |
|
| 7 |
import org.apache.commons.logging.Log; |
|
| 8 |
import org.apache.commons.logging.LogFactory; |
|
| 9 |
import org.apache.solr.client.solrj.impl.CloudSolrClient; |
|
| 10 |
import org.dom4j.Document; |
|
| 11 |
import org.dom4j.DocumentException; |
|
| 12 |
import org.dom4j.io.SAXReader; |
|
| 13 |
import org.springframework.beans.factory.annotation.Autowired; |
|
| 14 |
import org.springframework.beans.factory.annotation.Required; |
|
| 15 |
|
|
| 16 |
import com.google.common.collect.Maps; |
|
| 17 |
|
|
| 8 | 18 |
import eu.dnetlib.clients.index.model.Any.ValueType; |
| 9 | 19 |
import eu.dnetlib.clients.index.query.IndexQueryFactory; |
| 10 |
import eu.dnetlib.utils.MetadataReference;
|
|
| 20 |
import eu.dnetlib.cql.CqlValueTransformerMap;
|
|
| 11 | 21 |
import eu.dnetlib.index.AbstractBackendDescriptor; |
| 12 | 22 |
import eu.dnetlib.index.IndexCollection; |
| 13 | 23 |
import eu.dnetlib.index.IndexServerDAO; |
| 14 |
import eu.dnetlib.cql.CqlValueTransformerMap; |
|
| 15 | 24 |
import eu.dnetlib.index.feed.DocumentMapperFactory; |
| 16 | 25 |
import eu.dnetlib.index.query.SolrIndexQueryFactory; |
| 17 | 26 |
import eu.dnetlib.index.query.SolrIndexQueryResponseFactory; |
| ... | ... | |
| 19 | 28 |
import eu.dnetlib.index.solr.feed.SolrDocumentMapperFactory; |
| 20 | 29 |
import eu.dnetlib.index.utils.IndexConfigFactory; |
| 21 | 30 |
import eu.dnetlib.index.utils.RemoteSolrAdministrator; |
| 31 |
import eu.dnetlib.index.utils.ZkServers; |
|
| 22 | 32 |
import eu.dnetlib.index.utils.ZkUtils; |
| 23 | 33 |
import eu.dnetlib.rmi.provision.IndexServiceException; |
| 24 |
import org.apache.commons.logging.Log; |
|
| 25 |
import org.apache.commons.logging.LogFactory; |
|
| 26 |
import org.apache.solr.client.solrj.SolrServerException; |
|
| 27 |
import org.apache.solr.client.solrj.impl.CloudSolrClient; |
|
| 28 |
import org.dom4j.Document; |
|
| 29 |
import org.dom4j.DocumentException; |
|
| 30 |
import org.dom4j.io.SAXReader; |
|
| 31 |
import org.springframework.beans.factory.annotation.Autowired; |
|
| 32 |
import org.springframework.beans.factory.annotation.Required; |
|
| 34 |
import eu.dnetlib.utils.MetadataReference; |
|
| 33 | 35 |
|
| 34 | 36 |
/** |
| 35 | 37 |
* The Class SolrIndexServerDAO. |
| ... | ... | |
| 41 | 43 |
*/ |
| 42 | 44 |
private static final Log log = LogFactory.getLog(SolrIndexServerDAO.class); // NOPMD by marko on 11/24/08 5:02 PM |
| 43 | 45 |
|
| 44 |
/** |
|
| 45 |
* The zk utils. |
|
| 46 |
*/ |
|
| 46 |
/** The zk utils. */ |
|
| 47 | 47 |
@Autowired |
| 48 | 48 |
private ZkUtils zkUtils; |
| 49 | 49 |
|
| 50 |
/** |
|
| 51 |
* The query response factory. |
|
| 52 |
*/ |
|
| 50 |
/** The query response factory. */ |
|
| 53 | 51 |
@Autowired |
| 54 | 52 |
private SolrIndexQueryResponseFactory queryResponseFactory; |
| 55 | 53 |
|
| 56 | 54 |
@Autowired |
| 57 | 55 |
private SolrIndexQueryFactory solrIndexQueryFactory; |
| 58 | 56 |
|
| 59 |
/** |
|
| 60 |
* The solr document mapper factory. |
|
| 61 |
*/ |
|
| 57 |
/** The solr document mapper factory. */ |
|
| 62 | 58 |
@Autowired |
| 63 | 59 |
private SolrDocumentMapperFactory solrDocumentMapperFactory; |
| 64 | 60 |
|
| 65 |
/** |
|
| 66 |
* The solr administrator. |
|
| 67 |
*/ |
|
| 61 |
/** The solr administrator. */ |
|
| 68 | 62 |
private RemoteSolrAdministrator solrAdministrator; |
| 69 | 63 |
|
| 70 |
/** |
|
| 71 |
* The solr type based cql value transformer map. |
|
| 72 |
*/ |
|
| 64 |
/** The solr type based cql value transformer map. */ |
|
| 73 | 65 |
@Autowired |
| 74 | 66 |
private SolrTypeBasedCqlValueTransformerMapFactory tMapFactory; |
| 75 | 67 |
|
| 76 | 68 |
/** |
| 77 | 69 |
* {@inheritDoc}
|
| 78 |
* <p> |
|
| 79 |
* String) |
|
| 70 |
* |
|
| 71 |
* @see eu.dnetlib.index.IndexServerDAO#createIndexCollection(eu.dnetlib.index.utils.MetadataReference, |
|
| 72 |
* java.lang.String) |
|
| 80 | 73 |
*/ |
| 81 | 74 |
@Override |
| 82 | 75 |
public void createIndexCollection(final MetadataReference mdref, final String fields) throws IndexServiceException {
|
| 83 |
CloudSolrClient client = null; |
|
| 84 |
try {
|
|
| 76 |
|
|
| 77 |
log.debug("createIndexCollection " + mdref.toString() + " " + fields );
|
|
| 78 |
|
|
| 79 |
try(CloudSolrClient client = getClient()) {
|
|
| 85 | 80 |
|
| 86 |
client = getClient(); |
|
| 87 | 81 |
client.connect(); |
| 88 |
final Map<String, String> p = getServiceProperties(); |
|
| 89 | 82 |
|
| 90 |
if (!solrAdministrator.indexCollectionExists(mdref.toString(), client, p.get("host"), p.get("port"))) {
|
|
| 91 |
Map<String, String> params = new HashMap<>();
|
|
| 83 |
if (!solrAdministrator.indexCollectionExists(mdref.toString(), client)) {
|
|
| 84 |
Map<String, String> params = Maps.newHashMap();
|
|
| 92 | 85 |
|
| 86 |
final Map<String, String> p = getServiceProperties(); |
|
| 93 | 87 |
params.put("numShards", p.get("numShards"));
|
| 94 | 88 |
params.put("replicationFactor", p.get("replicationFactor"));
|
| 95 | 89 |
|
| 96 | 90 |
for (IndexConfigFactory.CONFIG_PARAMS param_Name : IndexConfigFactory.CONFIG_PARAMS.values()) {
|
| 97 | 91 |
params.put(param_Name.toString(), p.get(param_Name.toString())); |
| 98 | 92 |
} |
| 99 |
params.put(IndexConfigFactory.CONFIG_PARAMS.indexDataDir.toString(), mdref.toString()); |
|
| 100 | 93 |
|
| 101 |
zkUtils.uploadZookeperConfig(client.getZkStateReader().getZkClient(), mdref.toString(), parse(fields), params, false);
|
|
| 94 |
zkUtils.uploadZookeperConfig(client.getZkStateReader().getZkClient(), mdref.toString(), parse(fields), params, true);
|
|
| 102 | 95 |
solrAdministrator.createSolrIndex(p.get("host"), p.get("port"), mdref.toString(), p.get("numShards"), p.get("replicationFactor"),
|
| 103 |
mdref.toString()); |
|
| 96 |
mdref.toString(), p.get("maxShardsPerNode"));
|
|
| 104 | 97 |
} |
| 105 | 98 |
client.getZkStateReader().close(); |
| 106 | 99 |
|
| 107 | 100 |
} catch (Exception e) {
|
| 108 | 101 |
log.error("Error on creating IndexCollection", e);
|
| 109 | 102 |
throw new IndexServiceException("Error on creating IndexCollection", e);
|
| 110 |
} finally {
|
|
| 111 |
if (client != null) {
|
|
| 112 |
try {
|
|
| 113 |
client.close(); |
|
| 114 |
} catch (IOException e) {
|
|
| 115 |
throw new IndexServiceException("Error while closing client", e);
|
|
| 116 |
} |
|
| 117 |
} |
|
| 118 | 103 |
} |
| 119 | 104 |
} |
| 120 | 105 |
|
| 121 | 106 |
@Override |
| 122 | 107 |
public void updateIndexCollection(final MetadataReference mdRef, final Document fields) throws IndexServiceException {
|
| 123 |
CloudSolrClient client = null; |
|
| 124 |
try {
|
|
| 125 |
client = getClient(); |
|
| 108 |
try(CloudSolrClient client = getClient()) {
|
|
| 109 |
|
|
| 126 | 110 |
client.connect(); |
| 127 |
Map<String, String> params = new HashMap<>();
|
|
| 111 |
Map<String, String> params = Maps.newHashMap();
|
|
| 128 | 112 |
|
| 129 | 113 |
params.put("numShards", getServiceProperties().get("numShards"));
|
| 130 | 114 |
params.put("replicationFactor", getServiceProperties().get("replicationFactor"));
|
| ... | ... | |
| 132 | 116 |
for (IndexConfigFactory.CONFIG_PARAMS param_Name : IndexConfigFactory.CONFIG_PARAMS.values()) {
|
| 133 | 117 |
params.put(param_Name.toString(), getServiceProperties().get(param_Name.toString())); |
| 134 | 118 |
} |
| 135 |
params.put(IndexConfigFactory.CONFIG_PARAMS.indexDataDir.toString(), mdRef.toString()); |
|
| 119 |
|
|
| 136 | 120 |
zkUtils.uploadZookeperConfig(client.getZkStateReader().getZkClient(), mdRef.toString(), fields, params, true); |
| 137 | 121 |
client.getZkStateReader().close(); |
| 138 |
client.shutdown(); |
|
| 139 | 122 |
solrAdministrator.reloadCollection(getServiceProperties().get("host"), getServiceProperties().get("port"), mdRef.toString());
|
| 140 | 123 |
|
| 141 | 124 |
} catch (Exception e) {
|
| 142 | 125 |
log.error("Error on updating IndexCollection", e);
|
| 143 | 126 |
throw new IndexServiceException("Error on updating IndexCollection", e);
|
| 144 |
} finally {
|
|
| 145 |
if (client != null) {
|
|
| 146 |
client.shutdown(); |
|
| 147 |
} |
|
| 148 | 127 |
} |
| 149 | 128 |
} |
| 150 | 129 |
|
| 151 | 130 |
/** |
| 152 | 131 |
* Parses the fields parameter. |
| 153 |
* |
|
| 154 |
* @param fields the fields |
|
| 132 |
* |
|
| 133 |
* @param fields |
|
| 134 |
* the fields |
|
| 155 | 135 |
* @return the document |
| 156 |
* @throws IndexServiceException the index service exception |
|
| 136 |
* @throws IndexServiceException |
|
| 137 |
* the index service exception |
|
| 157 | 138 |
*/ |
| 158 | 139 |
private Document parse(final String fields) throws IndexServiceException {
|
| 159 | 140 |
try {
|
| ... | ... | |
| 165 | 146 |
|
| 166 | 147 |
/** |
| 167 | 148 |
* {@inheritDoc}
|
| 149 |
* |
|
| 150 |
* @see eu.dnetlib.index.IndexServerDAO#getIndexCollection(eu.dnetlib.index.utils.MetadataReference) |
|
| 168 | 151 |
*/ |
| 169 | 152 |
@Override |
| 170 | 153 |
public IndexCollection getIndexCollection(final MetadataReference mdref) throws IndexServiceException {
|
| 171 |
CloudSolrClient newServer = getClient(mdref); |
|
| 172 |
try {
|
|
| 173 |
newServer.ping(); |
|
| 174 |
} catch (SolrServerException e) {
|
|
| 175 |
e.printStackTrace(); |
|
| 176 |
} catch (IOException e) {
|
|
| 177 |
e.printStackTrace(); |
|
| 178 |
} |
|
| 179 |
return new SolrIndexCollection(newServer); |
|
| 154 |
CloudSolrClient client = getClient(mdref); |
|
| 155 |
return new SolrIndexCollection(client); |
|
| 180 | 156 |
} |
| 181 | 157 |
|
| 182 | 158 |
/** |
| 183 | 159 |
* {@inheritDoc}
|
| 160 |
* |
|
| 161 |
* @see eu.dnetlib.index.IndexServerDAO#getSchema(MetadataReference) |
|
| 184 | 162 |
*/ |
| 185 | 163 |
@Override |
| 186 | 164 |
public Map<String, ValueType> getSchema(final MetadataReference mdRef) throws IndexServiceException {
|
| 187 |
CloudSolrClient server = getClient(mdRef); |
|
| 188 |
Map<String, ValueType> fields = solrAdministrator.getFieldNamesAndTypes(mdRef.toString(), server); |
|
| 189 |
try {
|
|
| 190 |
server.close(); |
|
| 191 |
} catch (IOException e) {
|
|
| 192 |
throw new IndexServiceException("Error on closing client");
|
|
| 193 |
} |
|
| 165 |
CloudSolrClient client = getClient(mdRef); |
|
| 166 |
Map<String, ValueType> fields = solrAdministrator.getFieldNamesAndTypes(mdRef.toString(), client); |
|
| 167 |
shutdown(mdRef); |
|
| 194 | 168 |
return fields; |
| 195 | 169 |
} |
| 196 | 170 |
|
| 197 | 171 |
/** |
| 198 | 172 |
* {@inheritDoc}
|
| 173 |
* |
|
| 174 |
* @see eu.dnetlib.index.IndexServerDAO#getCqlValueTransformerMap(MetadataReference) |
|
| 199 | 175 |
*/ |
| 200 | 176 |
@Override |
| 201 | 177 |
public CqlValueTransformerMap getCqlValueTransformerMap(final MetadataReference mdRef) throws IndexServiceException {
|
| ... | ... | |
| 204 | 180 |
|
| 205 | 181 |
/** |
| 206 | 182 |
* {@inheritDoc}
|
| 183 |
* |
|
| 184 |
* @see eu.dnetlib.index.IndexServerDAO#getDocumentMapperFactory() |
|
| 207 | 185 |
*/ |
| 208 | 186 |
@Override |
| 209 | 187 |
public DocumentMapperFactory getDocumentMapperFactory() throws IndexServiceException {
|
| ... | ... | |
| 212 | 190 |
|
| 213 | 191 |
/** |
| 214 | 192 |
* {@inheritDoc}
|
| 193 |
* |
|
| 194 |
* @see eu.dnetlib.index.IndexServerDAO#shutdown(MetadataReference) |
|
| 215 | 195 |
*/ |
| 216 | 196 |
@Override |
| 217 | 197 |
public void shutdown(final MetadataReference mdRef) throws IndexServiceException {
|
| 218 | 198 |
try {
|
| 219 | 199 |
getClient(mdRef).close(); |
| 220 | 200 |
} catch (IOException e) {
|
| 221 |
throw new IndexServiceException("Error on closing client", e);
|
|
| 201 |
throw new IndexServiceException(e); |
|
| 222 | 202 |
} |
| 223 | 203 |
|
| 224 | 204 |
} |
| 225 | 205 |
|
| 226 | 206 |
/** |
| 227 | 207 |
* Gets a server with the default collection set according to the given mdRef. |
| 228 |
* |
|
| 229 |
* @param mdRef the md ref |
|
| 208 |
* |
|
| 209 |
* @param mdRef |
|
| 210 |
* the md ref |
|
| 230 | 211 |
* @return a server instance |
| 231 |
* @throws IndexServiceException the index service exception |
|
| 212 |
* @throws IndexServiceException |
|
| 213 |
* the index service exception |
|
| 232 | 214 |
*/ |
| 233 | 215 |
private CloudSolrClient getClient(final MetadataReference mdRef) throws IndexServiceException {
|
| 234 | 216 |
CloudSolrClient client = getClient(); |
| ... | ... | |
| 238 | 220 |
|
| 239 | 221 |
/** |
| 240 | 222 |
* Gets the server. |
| 241 |
* |
|
| 223 |
*
|
|
| 242 | 224 |
* @return a server instance |
| 243 | 225 |
*/ |
| 244 | 226 |
private CloudSolrClient getClient() {
|
| 245 | 227 |
String address = getEndpoint().get(ADDRESS); |
| 246 | 228 |
log.info("connecting to address: " + address);
|
| 247 |
return new CloudSolrClient(address); |
|
| 248 | 229 |
|
| 230 |
final ZkServers zk = ZkServers.newInstance(address); |
|
| 231 |
return new CloudSolrClient.Builder(zk.getHosts(), zk.getChroot()).build(); |
|
| 249 | 232 |
} |
| 250 | 233 |
|
| 251 | 234 |
/** |
| 252 | 235 |
* Gets the solr administrator. |
| 253 |
* |
|
| 236 |
*
|
|
| 254 | 237 |
* @return the solrAdministrator |
| 255 | 238 |
*/ |
| 256 | 239 |
public RemoteSolrAdministrator getSolrAdministrator() {
|
| ... | ... | |
| 259 | 242 |
|
| 260 | 243 |
/** |
| 261 | 244 |
* Sets the solr administrator. |
| 262 |
* |
|
| 263 |
* @param solrAdministrator the solrAdministrator to set |
|
| 245 |
* |
|
| 246 |
* @param solrAdministrator |
|
| 247 |
* the solrAdministrator to set |
|
| 264 | 248 |
*/ |
| 265 | 249 |
@Required |
| 266 | 250 |
public void setSolrAdministrator(final RemoteSolrAdministrator solrAdministrator) {
|
Also available in: Unified diff
solr 772 integration