Project

General

Profile

« Previous | Next » 

Revision 42853

Added by Claudio Atzori over 8 years ago

using soap method to obtain hadoop cluster configurations

View differences:

modules/dnet-openaire/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/PrepareStatsParamJobNode.java
1 1 
package eu.dnetlib.msro.workflows.nodes.hadoop;
2 2 

  		




  3
  
  
    
import javax.annotation.Resource;


  		




  
  3
  
    
import java.util.Map;


  		




  4
  4
  
    

  		




  
  5
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  
  6
  
    
import eu.dnetlib.rmi.data.hadoop.HadoopService;


  		




  5
  7
  
    
import org.apache.commons.logging.Log;


  		




  6
  8
  
    
import org.apache.commons.logging.LogFactory;


  		




  7
  9
  
    

  		




  ......




  12
  14
  
    
import eu.dnetlib.rmi.enabling.ISLookUpDocumentNotFoundException;


  		




  13
  15
  
    
import eu.dnetlib.rmi.enabling.ISLookUpException;


  		




  14
  16
  
    
import eu.dnetlib.rmi.enabling.ISLookUpService;


  		




  
  17
  
    
import org.springframework.beans.factory.annotation.Autowired;


  		




  15
  18
  
    

  		




  16
  19
  
    
public class PrepareStatsParamJobNode extends SimpleJobNode {


  		




  17
  20
  
    

  		




  ......




  23
  26
  
    
	private static final String xqueryIsLookupEndpoint =


  		




  24
  27
  
    
			"//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='IS_LookUpServiceResourceType']//PROTOCOL[./@name='SOAP']/@address/string()";


  		




  25
  28
  
    

  		




  26
  
  
    
	@Resource


  		




  
  29
  
    
	@Autowired


  		




  27
  30
  
    
	private UniqueServiceLocator serviceLocator;


  		




  28
  31
  
    

  		




  29
  
  
    
	@Resource


  		




  30
  
  
    
	protected ConfigurationEnumerator configurationEnumerator;


  		




  31
  
  
    

  		




  32
  32
  
    
	@Override


  		




  33
  33
  
    
	protected String execute(final Env env) throws Exception {


  		




  34
  34
  
    

  		




  35
  35
  
    
		log.info("prepare stats update job");


  		




  36
  36
  
    

  		




  37
  
  
    
		final Configuration conf = this.configurationEnumerator.get(ClusterName.DM);


  		




  
  37
  
    
		final Map<String, String> conf = serviceLocator.getService(HadoopService.class).getClusterConfiguration(ClusterName.DM.toString());


  		




  38
  38
  
    
		final String nameNode = conf.get("fs.defaultFS");


  		




  39
  39
  
    

  		




  40
  40
  
    
		env.setAttribute("nameNode", nameNode);


  		




  ......




  45
  45
  
    
		return Arc.DEFAULT_ARC;


  		




  46
  46
  
    
	}


  		




  47
  47
  
    

  		




  48
  
  
    
	private String getIsLookupEndpoint() throws ISLookUpDocumentNotFoundException, ISLookUpException {


  		




  
  48
  
    
	private String getIsLookupEndpoint() throws ISLookUpException {


  		




  49
  49
  
    
		return this.serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xqueryIsLookupEndpoint);


  		




  50
  50
  
    
	}


  		




  51
  51
  
    

  		












  

    
      modules/dnet-openaire/trunk/src/main/java/eu/dnetlib/msro/workflows/nodes/hadoop/PrepareIISParams.java
    
  





  1
  1
  
    
package eu.dnetlib.msro.workflows.nodes.hadoop;


  		




  2
  2
  
    

  		




  3
  3
  
    
import java.util.List;


  		




  
  4
  
    
import java.util.Map;


  		




  4
  5
  
    
import java.util.NoSuchElementException;


  		




  5
  6
  
    

  		




  6
  7
  
    
import javax.annotation.Resource;


  		




  7
  8
  
    

  		




  
  9
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  
  10
  
    
import eu.dnetlib.rmi.data.hadoop.HadoopService;


  		




  8
  11
  
    
import org.apache.commons.logging.Log;


  		




  9
  12
  
    
import org.apache.commons.logging.LogFactory;


  		




  10
  13
  
    
import org.springframework.beans.factory.annotation.Required;


  		




  ......




  26
  29
  
    
	private static final Log log = LogFactory.getLog(PrepareIISParams.class);


  		




  27
  30
  
    

  		




  28
  31
  
    
	@Resource


  		




  29
  
  
    
	protected ConfigurationEnumerator configurationEnumerator;


  		




  30
  
  
    

  		




  31
  
  
    
	@Resource


  		




  32
  32
  
    
	private UniqueServiceLocator serviceLocator;


  		




  33
  33
  
    

  		




  34
  34
  
    
	private String clusterName;


  		




  ......




  78
  78
  
    

  		




  79
  79
  
    
		env.setAttribute(getImportProjectConceptsContextIdParam(), getImportProjectConceptsContextId());


  		




  80
  80
  
    

  		




  81
  
  
    
		final Configuration conf = this.configurationEnumerator.get(ClusterName.valueOf(getClusterName()));


  		




  
  81
  
    
		final Map<String, String> conf = serviceLocator.getService(HadoopService.class).getClusterConfiguration(getClusterName());


  		




  82
  82
  
    

  		




  83
  
  
    
		final Configuration exportConf = this.configurationEnumerator.get(ClusterName.DM);


  		




  
  83
  
    
		final Map<String, String> exportConf = serviceLocator.getService(HadoopService.class).getClusterConfiguration(ClusterName.DM.toString());


  		




  
  84
  
    

  		




  84
  85
  
    
		final String zkPort = exportConf.get("hbase.zookeeper.property.clientPort");


  		




  85
  86
  
    
		final String zkQuorum = getZkQuorumCSV(exportConf, zkPort);


  		




  86
  87
  
    

  		




  ......




  156
  157
  
    
		return nameNode + relative;


  		




  157
  158
  
    
	}


  		




  158
  159
  
    

  		




  159
  
  
    
	private String getZkQuorumCSV(final Configuration conf, final String zkPort) {


  		




  
  160
  
    
	private String getZkQuorumCSV(final Map<String, String> conf, final String zkPort) {


  		




  160
  161
  
    
		return Joiner.on(":" + zkPort + ",").join(Splitter.on(",").omitEmptyStrings().split(conf.get("hbase.zookeeper.quorum")));


  		




  161
  162
  
    
	}


  		




  162
  163
  
    

  		












Also available in:  Unified diff