Project

General

Profile

« Previous | Next » 

Revision 29860

Added by Claudio Atzori about 10 years ago

changed properties passed to index feed m/r job

View differences:

modules/dnet-mapreduce-jobs/branches/newIndexFeed/src/main/java/eu/dnetlib/data/mapreduce/hbase/index/IndexFeedMapper.java
47 47 

  		




  48
  48
  
    
		logConfiguration(context.getConfiguration());


  		




  49
  49
  
    

  		




  50
  
  
    
		String baseURL = context.getConfiguration().get(JobParams.INDEX_SOLR_URL_LIST) + "/" + context.getConfiguration().get(JobParams.INDEX_SOLR_COLLECTION);


  		




  
  50
  
    
		String baseURL = context.getConfiguration().get(JobParams.INDEX_SOLR_URL);


  		




  51
  51
  
    

  		




  52
  52
  
    
		System.out.println("solr server baseURL: " + baseURL);


  		




  53
  53
  
    

  		




  ......




  84
  84
  
    
		try {


  		




  85
  85
  
    
			long start = System.currentTimeMillis();


  		




  86
  86
  
    
			indexRecord = dmfToRecord.evaluate(value.toString());


  		




  87
  
  
    
			doc = documentFactory.parseDocument(version, indexRecord.replaceAll("&#", "&#"), dsId);


  		




  
  87
  
    
			doc = documentFactory.parseDocument(version, indexRecord, dsId);


  		




  88
  88
  
    
			long stop = System.currentTimeMillis() - start;


  		




  89
  89
  
    

  		




  90
  90
  
    
			System.out.println("parse " + doc.getField("__indexrecordidentifier").getValue() + " : " + HumanTime.exactly(stop));


  		












  

    
      modules/dnet-openaireplus-workflows/branches/newIndexFeed/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/PrepareIndexDataJobNode.java
    
  





  4
  4
  
    
import java.io.InputStream;


  		




  5
  5
  
    
import java.io.StringReader;


  		




  6
  6
  
    
import java.io.StringWriter;


  		




  
  7
  
    
import java.net.URI;


  		




  7
  8
  
    
import java.util.List;


  		




  8
  9
  
    

  		




  9
  10
  
    
import javax.annotation.Resource;


  		




  ......




  70
  71
  
    
		if (!StringUtils.isBlank(getRottenRecordsPathParam())) {


  		




  71
  72
  
    
			token.getEnv().setAttribute(getRottenRecordsPathParam(), "/tmp" + getFileName(token, "rottenrecords"));


  		




  72
  73
  
    
		}


  		




  73
  
  
    
		token.getEnv().setAttribute("index.solr.url.zk", getIndexSolrUrlZk());


  		




  74
  
  
    
		token.getEnv().setAttribute("index.solr.url.local", getIndexSolrUrlLocal());


  		




  75
  
  
    
		token.getEnv().setAttribute("index.solr.url.list", getIndexSolrUrlList());


  		




  76
  74
  
    

  		




  
  75
  
    
		token.getEnv().setAttribute("index.solr.url", getIndexSolrUrl(token));


  		




  
  76
  
    

  		




  77
  77
  
    
		token.getEnv().setAttribute("index.shutdown.wait.time", getIndexSolrShutdownWait());


  		




  78
  78
  
    
		token.getEnv().setAttribute("index.buffer.flush.threshold", getIndexBufferFlushTreshold());


  		




  79
  
  
    
		token.getEnv().setAttribute("index.solr.local.feeding", isIndexLocalFeeding());


  		




  80
  79
  
    
		token.getEnv().setAttribute("index.solr.sim.mode", isFeedingSimulationMode());


  		




  81
  80
  
    

  		




  82
  
  
    
		token.getEnv().setAttribute("index.solr.collection", getCollectionName(token));


  		




  83
  81
  
    
		token.getEnv().setAttribute("index.feed.timestamp", DateUtils.now_ISO8601());


  		




  84
  82
  
    

  		




  85
  83
  
    
		token.getEnv().setAttribute(getOafSchemaLocationProperty(), getPropertyFetcher().getProperty(getOafSchemaLocationProperty()));


  		




  ......




  87
  85
  
    
		return Arc.DEFAULT_ARC;


  		




  88
  86
  
    
	}


  		




  89
  87
  
    

  		




  90
  
  
    
	public String getIndexSolrUrlZk() throws ISLookUpException {


  		




  91
  
  
    
		return getServiceConfigValue("for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='IndexServiceResourceType'] return $x//PROTOCOL[./@name='SOLR']/@address/string()");


  		




  
  88
  
    
	private String getIndexSolrUrl(final NodeToken token) throws ISLookUpException {


  		




  
  89
  
    
		String baseUrl = "http://" + getSolrServerHost() + ":" + getSolrServerPort() + "/" + getSolrWebappContext() + "/" + getCollectionName(token);


  		




  
  90
  
    
		URI uri = URI.create(baseUrl);


  		




  
  91
  
    
		return uri.toString();


  		




  92
  92
  
    
	}


  		




  93
  93
  
    

  		




  94
  
  
    
	public String getIndexSolrUrlLocal() throws ISLookUpException {


  		




  95
  
  
    
		return queryForServiceProperty("hadoopLocalUrl");


  		




  
  94
  
    
	private String getSolrServerHost() throws ISLookUpException {


  		




  
  95
  
    
		return queryForServiceProperty("solr:host");


  		




  96
  96
  
    
	}


  		




  97
  97
  
    

  		




  98
  
  
    
	public String getIndexSolrUrlList() throws ISLookUpException {


  		




  99
  
  
    
		return queryForServiceProperty("clientPoolUrls");


  		




  
  98
  
    
	private String getSolrServerPort() throws ISLookUpException {


  		




  
  99
  
    
		return queryForServiceProperty("solr:port");


  		




  100
  100
  
    
	}


  		




  101
  101
  
    

  		




  
  102
  
    
	private String getSolrWebappContext() throws ISLookUpException {


  		




  
  103
  
    
		return queryForServiceProperty("solr:webContext");


  		




  
  104
  
    
	}


  		




  
  105
  
    

  		




  102
  106
  
    
	public String getIndexSolrShutdownWait() throws ISLookUpException {


  		




  103
  107
  
    
		return queryForServiceProperty("feedingShutdownTolerance");


  		




  104
  108
  
    
	}


  		




  ......




  123
  127
  
    
	private String getServiceConfigValue(final String xquery) throws ISLookUpException {


  		




  124
  128
  
    
		log.debug("quering for service property: " + xquery);


  		




  125
  129
  
    
		final List<String> urls = lookupLocator.getService().quickSearchProfile(xquery);


  		




  126
  
  
    
		if ((urls == null) || (urls.size() != 1)) throw new IllegalStateException("unable to find unique service property, xquery: " + xquery);


  		




  
  130
  
    
		if ((urls == null) || (urls.size() != 1)) { throw new IllegalStateException("unable to find unique service property, xquery: " + xquery); }


  		




  127
  131
  
    
		return Iterables.getOnlyElement(urls);


  		




  128
  132
  
    
	}


  		




  129
  133
  
    

  		












Also available in:  Unified diff