Project

General

Profile

« Previous | Next » 

Revision 46709

Added by Claudio Atzori about 7 years ago

integrated changes from dnet40: added support for create multiple shards for the same collection on the same solr server

View differences:

modules/dnet-index-solr-service/trunk/src/main/java/eu/dnetlib/functionality/index/utils/IndexConfigFactory.java
25 25 

  		




  26
  26
  
    
	public enum CONFIG_PARAMS {


  		




  27
  27
  
    
		// TODO replace with actual values


  		




  28
  
  
    
		luceneMatchVersion, serverLibPath, indexDataDir, filterCacheSize, filterCacheInitialSize, queryCacheSize, queryCacheInitialSize, documentCacheSize, documentCacheInitialSize, queryResultWindowSize, queryResultMaxDocCached, ramBufferSizeMB, mergeFactor, termIndexInterval, autosoftcommit, autocommit, maxIndexingThreads;


  		




  
  28
  
    
		luceneMatchVersion,


  		




  
  29
  
    
		serverLibPath,


  		




  
  30
  
    
		filterCacheSize,


  		




  
  31
  
    
		filterCacheInitialSize,


  		




  
  32
  
    
		queryCacheSize,


  		




  
  33
  
    
		queryCacheInitialSize,


  		




  
  34
  
    
		documentCacheSize,


  		




  
  35
  
    
		documentCacheInitialSize,


  		




  
  36
  
    
		queryResultWindowSize,


  		




  
  37
  
    
		queryResultMaxDocCached,


  		




  
  38
  
    
		ramBufferSizeMB,


  		




  
  39
  
    
		mergeFactor,


  		




  
  40
  
    
		termIndexInterval,


  		




  
  41
  
    
		autosoftcommit,


  		




  
  42
  
    
		autocommit,


  		




  
  43
  
    
		maxIndexingThreads;


  		




  29
  44
  
    

  		




  30
  45
  
    
		static boolean isValid(final String v) {


  		




  31
  46
  
    
			try {


  		












  

    
      modules/dnet-index-solr-service/trunk/src/main/java/eu/dnetlib/functionality/index/utils/RemoteSolrAdministrator.java
    
  





  1
  1
  
    
package eu.dnetlib.functionality.index.utils;


  		




  2
  2
  
    

  		




  3
  3
  
    
import java.io.IOException;


  		




  4
  
  
    
import java.util.HashMap;


  		




  5
  4
  
    
import java.util.Map;


  		




  6
  5
  
    

  		




  
  6
  
    
import com.google.common.collect.Maps;


  		




  
  7
  
    
import com.google.gson.JsonElement;


  		




  
  8
  
    
import com.google.gson.JsonObject;


  		




  
  9
  
    
import com.google.gson.JsonParser;


  		




  
  10
  
    
import eu.dnetlib.data.provision.index.rmi.IndexServiceException;


  		




  
  11
  
    
import eu.dnetlib.functionality.index.model.Any.ValueType;


  		




  7
  12
  
    
import org.apache.commons.logging.Log;


  		




  8
  13
  
    
import org.apache.commons.logging.LogFactory;


  		




  9
  14
  
    
import org.apache.http.HttpResponse;


  		




  ......




  22
  27
  
    
import org.apache.zookeeper.data.Stat;


  		




  23
  28
  
    
import org.springframework.beans.factory.annotation.Required;


  		




  24
  29
  
    

  		




  25
  
  
    
import com.google.common.collect.Maps;


  		




  26
  
  
    
import com.google.gson.JsonElement;


  		




  27
  
  
    
import com.google.gson.JsonObject;


  		




  28
  
  
    
import com.google.gson.JsonParser;


  		




  29
  
  
    

  		




  30
  
  
    
import eu.dnetlib.data.provision.index.rmi.IndexServiceException;


  		




  31
  
  
    
import eu.dnetlib.functionality.index.model.Any.ValueType;


  		




  32
  
  
    

  		




  33
  30
  
    
public class RemoteSolrAdministrator {


  		




  34
  31
  
    

  		




  35
  32
  
    
	/**


  		




  ......




  38
  35
  
    
	private static final Log log = LogFactory.getLog(RemoteSolrAdministrator.class);


  		




  39
  36
  
    

  		




  40
  37
  
    
	/** The create url request. */


  		




  41
  
  
    
	private static String createURLRequest = "http://%s:%s/solr/admin/collections?action=CREATE&name=%s&numShards=%s&replicationFactor=%s&collection.configName=%s";


  		




  
  38
  
    
	private static String createURLRequest = "http://%s:%s/solr/admin/collections?action=CREATE&name=%s&numShards=%s&replicationFactor=%s&maxShardsPerNode=%s&collection.configName=%s";


  		




  42
  39
  
    

  		




  43
  40
  
    
	/** The create url request. */


  		




  44
  41
  
    
	private static String reloadURLRequest = "http://%s:%s/solr/admin/collections?action=RELOAD&name=%s";


  		




  ......




  49
  46
  
    
	protected Map<String, Map<String, ValueType>> cachedSchema;


  		




  50
  47
  
    

  		




  51
  48
  
    
	public RemoteSolrAdministrator() {


  		




  52
  
  
    
		this.cachedSchema = new HashMap<String, Map<String, ValueType>>();


  		




  
  49
  
    
		this.cachedSchema = Maps.newHashMap();


  		




  53
  50
  
    
	}


  		




  54
  51
  
    

  		




  55
  52
  
    
	public boolean createSolrIndex(final String host,


  		




  ......




  57
  54
  
    
			final String collectionName,


  		




  58
  55
  
    
			final String numShard,


  		




  59
  56
  
    
			final String replicationFactor,


  		




  60
  
  
    
			final String collectionConfigName) throws IndexServiceException {


  		




  
  57
  
    
			final String maxShardsPerNode,


  		




  
  58
  
    
		final String collectionConfigName) throws IndexServiceException {


  		




  61
  59
  
    

  		




  62
  
  
    
		final String uri = generateCreateIndexRequest(host, port, collectionName, numShard, replicationFactor, collectionConfigName);


  		




  
  60
  
    
		final String uri = generateCreateIndexRequest(host, port, collectionName, numShard, replicationFactor, maxShardsPerNode, collectionConfigName);


  		




  63
  61
  
    
		log.info(uri);


  		




  64
  62
  
    
		HttpGet request = new HttpGet(uri);


  		




  65
  63
  
    
		HttpResponse response;


  		




  ......




  156
  154
  
    
			final String collectionName,


  		




  157
  155
  
    
			final String numShard,


  		




  158
  156
  
    
			final String replicationFactor,


  		




  159
  
  
    
			final String collectionConfigName) {


  		




  160
  
  
    
		return String.format(createURLRequest, host, port, collectionName, numShard, replicationFactor, collectionConfigName);


  		




  
  157
  
    
			final String collectionConfigName,


  		




  
  158
  
    
			final String maxShardsPerNode) {


  		




  
  159
  
    
		return String.format(createURLRequest, host, port, collectionName, numShard, replicationFactor, maxShardsPerNode, collectionConfigName);


  		




  161
  160
  
    
	}


  		




  162
  161
  
    

  		




  163
  162
  
    
	private String generateUpdateIndexRequest(final String host, final String port, final String collectionName) {


  		












  

    
      modules/dnet-index-solr-service/trunk/src/main/java/eu/dnetlib/functionality/index/SolrIndexServerDAO.java
    
  





  85
  85
  
    
				for (IndexConfigFactory.CONFIG_PARAMS param_Name : IndexConfigFactory.CONFIG_PARAMS.values()) {


  		




  86
  86
  
    
					params.put(param_Name.toString(), p.get(param_Name.toString()));


  		




  87
  87
  
    
				}


  		




  88
  
  
    
				params.put(IndexConfigFactory.CONFIG_PARAMS.indexDataDir.toString(), mdref.toString());


  		




  89
  88
  
    

  		




  90
  89
  
    
				zkUtils.uploadZookeperConfig(server.getZkStateReader().getZkClient(), mdref.toString(), parse(fields), params, false);


  		




  91
  90
  
    
				solrAdministrator.createSolrIndex(p.get("host"), p.get("port"), mdref.toString(), p.get("numShards"), p.get("replicationFactor"),


  		




  92
  
  
    
						mdref.toString());


  		




  
  91
  
    
						mdref.toString(), p.get("maxShardsPerNode"));


  		




  93
  92
  
    
			}


  		




  94
  93
  
    
			server.getZkStateReader().close();


  		




  95
  94
  
    

  		




  ......




  117
  116
  
    
			for (IndexConfigFactory.CONFIG_PARAMS param_Name : IndexConfigFactory.CONFIG_PARAMS.values()) {


  		




  118
  117
  
    
				params.put(param_Name.toString(), getServiceProperties().get(param_Name.toString()));


  		




  119
  118
  
    
			}


  		




  120
  
  
    
			params.put(IndexConfigFactory.CONFIG_PARAMS.indexDataDir.toString(), mdRef.toString());


  		




  
  119
  
    

  		




  121
  120
  
    
			zkUtils.uploadZookeperConfig(server.getZkStateReader().getZkClient(), mdRef.toString(), fields, params, true);


  		




  122
  121
  
    
			server.getZkStateReader().close();


  		




  123
  122
  
    
			server.shutdown();


  		












  

    
      modules/dnet-index-solr-service/trunk/src/main/resources/eu/dnetlib/functionality/index/applicationContext-index-solr-service.properties
    
  





  6
  6
  
    
	"webContext":"solr",\


  		




  7
  7
  
    
	"numShards":"1",\


  		




  8
  8
  
    
	"replicationFactor":"1",\


  		




  
  9
  
    
    "maxShardsPerNode":"2",\


  		




  9
  10
  
    
	"host":"localhost",\


  		




  10
  11
  
    
	"feedingShutdownTolerance":"30000",\


  		




  11
  12
  
    
	"feedingBufferFlushThreshold":"1000",\


  		




  ......




  29
  30
  
    
services.mapreduce.index.solr.feed.sim.mode	= false


  		




  30
  31
  
    
service.index.solr.document.factory			= eu.dnetlib.functionality.index.solr.feed.StreamingInputDocumentFactory


  		




  31
  32
  
    
service.index.solr.schema.template			= classpath:/eu/dnetlib/functionality/index/conf/schemaTemplate.xslt


  		




  32
  
  
    

  		




  33
  
  
    

  		




  34
  
  
    
#service.index.configuration.backend.json	=


  		




  35
  
  
    
#service.index.solr.querycomponent 			= org.apache.solr.handler.component.QueryComponent


  		




  36
  
  
    
#service.index.solr.home.dir					=${dnet.data.path}/solr


  		




  37
  
  
    
#service.index.solr.data.dir					=${service.index.solr.home.dir}/data


  		




  38
  
  
    
#service.index.solr.overwrite.conf			=true


  		




  39
  
  
    
#service.index.solr.clustering.enabled		=false


  		




  40
  
  
    
#service.index.solr.conf.version				=LUCENE_41


  		




  41
  
  
    
#service.index.solr.optimization.cron		=0 59 23 * * ?


  		




  42
  
  
    
#service.index.solr.optimization.enable		=true


  		




  43
  
  
    
#service.index.solr.optimization.spacemargin =0.05


  		




  44
  
  
    
#service.index.solr.browse.limit				=1000


  		




  45
  
  
    
#service.index.solr.default.interpretation	=DRIVER


  		




  46
  
  
    
#service.index.solr.resultset.client.pagesize = 100


  		




  47
  
  
    
#service.index.solr.querycomponent 			= eu.dnetlib.functionality.index.solr.component.DynamicSearchComponent


  		




  48
  
  
    
#service.index.solr.create.onfeed			= false


  		




  49
  
  
    
#service.index.solr.embedded					= true


  		




  50
  
  
    
#service.index.solr.shards.tolerant			= false


  		




  51
  
  
    
#services.index.solr.remote.queue.size 		= 100


  		




  52
  
  
    
#services.index.solr.local.feeding 			= true


  		




  53
  
  
    
#services.index.solr.feed.sim.mode 			= false


  		




  54
  
  
    
#services.index.solr.remote.thread.count 	= 1


  		




  55
  
  
    
#service.index.solr.numshards				= 12


  		




  56
  
  
    
#service.index.solr.replicationfactor		= 1


  		




  57
  
  
    
#service.index.solr.conf.tmp.basedir			= /tmp/solr


  		












  

    
      modules/dnet-index-solr-service/trunk/src/main/resources/eu/dnetlib/functionality/index/conf/solrconfig.xml.st
    
  





  89
  89
  
    
       replication is in use, this should match the replication


  		




  90
  90
  
    
       configuration.


  		




  91
  91
  
    
    -->


  		




  92
  
  
    
  <dataDir>solr/$indexDataDir$</dataDir>


  		




  
  92
  
    
  <dataDir>\${solr.data.dir:}</dataDir>


  		




  93
  93
  
    

  		




  94
  94
  
    

  		




  95
  95
  
    
  <!-- The DirectoryFactory to use for indexes.


  		












Also available in:  Unified diff