Project

General

Profile

« Previous | Next » 

Revision 42937

Added by Claudio Atzori over 8 years ago

first migration on java8

View differences:

modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/actionmanager/blackboard/ActionManagerJobLauncher.java
19 19 
import eu.dnetlib.enabling.tools.blackboard.BlackboardClientHandler;
20 20 
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
21 21 
import eu.dnetlib.enabling.tools.blackboard.BlackboardJobRegistry;
22 
import eu.dnetlib.rmi.data.hadoop.ClusterName;
22 23 
import eu.dnetlib.rmi.data.hadoop.HadoopBlackboardActions;
23 24 
import eu.dnetlib.rmi.data.hadoop.HadoopService;
24 25 
import eu.dnetlib.rmi.data.hadoop.HadoopServiceException;
modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/actionmanager/blackboard/GarbageActionManagerActionFromHDFS.java
10 10 
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerAction;
11 11 
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
12 12 
import eu.dnetlib.miscutils.datetime.DateUtils;
13 
import eu.dnetlib.rmi.data.hadoop.ClusterName;
13 14 
import eu.dnetlib.rmi.data.hadoop.HadoopService;
14 15 
import eu.dnetlib.rmi.data.hadoop.actionmanager.ActionManagerSet;
15 16 
import eu.dnetlib.rmi.data.hadoop.actionmanager.RawSet;
modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/actionmanager/ActionManagerServiceImpl.java
6 6 
import eu.dnetlib.data.actionmanager.is.ISClient;
7 7 
import eu.dnetlib.data.hadoop.HadoopServiceCore;
8 8 

  		




  
  9
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  9
  10
  
    
import eu.dnetlib.rmi.data.hadoop.HadoopServiceException;


  		




  10
  11
  
    
import eu.dnetlib.enabling.tools.AbstractBaseService;


  		




  11
  12
  
    
import eu.dnetlib.enabling.tools.blackboard.NotificationHandler;


  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/utils/HadoopUtils.java
    
  





  3
  3
  
    
import java.util.Map.Entry;


  		




  4
  4
  
    
import java.util.Set;


  		




  5
  5
  
    

  		




  
  6
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  6
  7
  
    
import org.apache.hadoop.hbase.HColumnDescriptor;


  		




  7
  8
  
    
import org.apache.hadoop.hbase.HTableDescriptor;


  		




  8
  9
  
    

  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/hbase/HBaseAdminFactory.java
    
  





  1
  1
  
    
package eu.dnetlib.data.hadoop.hbase;


  		




  2
  2
  
    

  		




  
  3
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  3
  4
  
    
import org.apache.commons.logging.Log;


  		




  4
  5
  
    
import org.apache.commons.logging.LogFactory;


  		




  5
  6
  
    
import org.apache.hadoop.hbase.client.HBaseAdmin;


  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/hbase/HbaseTableFeeder.java
    
  





  12
  12
  
    
import eu.dnetlib.data.transform.XsltRowTransformerFactory;


  		




  13
  13
  
    
import eu.dnetlib.enabling.resultset.client.ResultSetClient;


  		




  14
  14
  
    
import eu.dnetlib.rmi.common.ResultSet;


  		




  
  15
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  15
  16
  
    
import org.apache.commons.logging.Log;


  		




  16
  17
  
    
import org.apache.commons.logging.LogFactory;


  		




  17
  18
  
    
import org.apache.hadoop.conf.Configuration;


  		




  ......




  56
  57
  
    
	/**


  		




  57
  58
  
    
	 * Feed.


  		




  58
  59
  
    
	 *


  		




  59
  
  
    
	 * @param epr         the epr


  		




  
  60
  
    
	 * @param rsJson      the resultSet


  		




  60
  61
  
    
	 * @param xsl         the xsl


  		




  61
  62
  
    
	 * @param clusterName the cluster name


  		




  62
  63
  
    
	 * @param tableName   the table name


  		




  ......




  65
  66
  
    
	 * @throws IOException          Signals that an I/O exception has occurred.


  		




  66
  67
  
    
	 * @throws InterruptedException the interrupted exception


  		




  67
  68
  
    
	 */


  		




  68
  
  
    
	public int feed(final String epr, final String xsl, final ClusterName clusterName, final String tableName, final boolean simulation)


  		




  
  69
  
    
	public int feed(final String rsJson, final String xsl, final ClusterName clusterName, final String tableName, final boolean simulation)


  		




  69
  70
  
    
			throws IOException, InterruptedException {


  		




  70
  
  
    
		return doWrite(asRows(epr, xsl), getConf(clusterName), tableName, simulation);


  		




  
  71
  
    
		return doWrite(asRows(rsJson, xsl), getConf(clusterName), tableName, simulation);


  		




  71
  72
  
    
	}


  		




  72
  73
  
    

  		




  73
  74
  
    
	/**


  		




  ......




  125
  126
  
    
	/**


  		




  126
  127
  
    
	 * As rows.


  		




  127
  128
  
    
	 *


  		




  128
  
  
    
	 * @param eprJson the epr


  		




  
  129
  
    
	 * @param rsJson the resultSet


  		




  129
  130
  
    
	 * @param xsl the xsl


  		




  130
  131
  
    
	 * @return the iterable


  		




  131
  132
  
    
	 */


  		




  132
  
  
    
	protected Iterable<Row> asRows(final String eprJson, final String xsl) {


  		




  133
  
  
    
		final ResultSet<String> resultSet = (ResultSet<String>) ResultSet.fromJson(eprJson);


  		




  
  133
  
    
	protected Iterable<Row> asRows(final String rsJson, final String xsl) {


  		




  
  134
  
    
		final ResultSet<String> resultSet = (ResultSet<String>) ResultSet.fromJson(rsJson);


  		




  134
  135
  
    
		return Iterables.concat(Iterables.transform(resultSetClient.iter(resultSet, String.class), XsltRowTransformerFactory.newInstance(xsl)));


  		




  135
  136
  
    
	}


  		




  136
  137
  
    

  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/JobRegistry.java
    
  





  5
  5
  
    
import java.util.Map;


  		




  6
  6
  
    
import java.util.Map.Entry;


  		




  7
  7
  
    

  		




  
  8
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  8
  9
  
    
import org.apache.commons.logging.Log;


  		




  9
  10
  
    
import org.apache.commons.logging.LogFactory;


  		




  10
  11
  
    
import org.springframework.beans.factory.annotation.Required;


  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/HadoopServiceCore.java
    
  





  14
  14
  
    
import com.google.common.collect.Maps;


  		




  15
  15
  
    
import com.google.common.collect.Sets;


  		




  16
  16
  
    
import eu.dnetlib.data.hadoop.config.ConfigurationEnumerator;


  		




  
  17
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  17
  18
  
    
import eu.dnetlib.rmi.data.hadoop.HadoopServiceException;


  		




  18
  19
  
    
import eu.dnetlib.rmi.data.hadoop.hbase.Column;


  		




  19
  20
  
    
import eu.dnetlib.rmi.data.hadoop.hbase.HBaseRowDescriptor;


  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/blackboard/DeleteHdfsPathAction.java
    
  





  1
  1
  
    
package eu.dnetlib.data.hadoop.blackboard;


  		




  2
  2
  
    

  		




  3
  3
  
    
import eu.dnetlib.data.hadoop.HadoopServiceCore;


  		




  
  4
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  4
  5
  
    
import eu.dnetlib.rmi.data.hadoop.HadoopServiceException;


  		




  5
  6
  
    
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;


  		




  6
  7
  
    
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;


  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/blackboard/ImportEprHbaseAction.java
    
  





  1
  1
  
    
package eu.dnetlib.data.hadoop.blackboard;


  		




  2
  2
  
    

  		




  
  3
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  3
  4
  
    
import org.apache.commons.logging.Log;


  		




  4
  5
  
    
import org.apache.commons.logging.LogFactory;


  		




  5
  6
  
    

  		




  ......




  19
  20
  
    
	@Override


  		




  20
  21
  
    
	public void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws HadoopServiceException {


  		




  21
  22
  
    

  		




  22
  
  
    
		final String eprJson = DnetXsltFunctions.decodeBase64(job.getParameters().get("input_epr"));


  		




  
  23
  
    
		final String rsJson = job.getParameters().get("input_epr");


  		




  23
  24
  
    
		final String xsl = DnetXsltFunctions.decodeBase64(job.getParameters().get("xslt"));


  		




  24
  25
  
    
		final ClusterName clusterName = ClusterName.valueOf(job.getParameters().get("cluster"));


  		




  25
  26
  
    
		final String table = job.getParameters().get("table");


  		




  ......




  28
  29
  
    
		log.info("Starting import in table " + table);


  		




  29
  30
  
    

  		




  30
  31
  
    
		try {


  		




  31
  
  
    
			final Integer count = getHbaseTableFeeder().feed(eprJson, xsl, clusterName, table, simulation);


  		




  
  32
  
    
			final Integer count = getHbaseTableFeeder().feed(rsJson, xsl, clusterName, table, simulation);


  		




  32
  33
  
    

  		




  33
  34
  
    
			if (count != null) {


  		




  34
  35
  
    
				log.info("Import completed successfully");


  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/blackboard/SubmitMapreduceJobAction.java
    
  





  6
  6
  
    
import java.util.Map.Entry;


  		




  7
  7
  
    

  		




  8
  8
  
    
import eu.dnetlib.miscutils.functional.xml.XMLIndenter;


  		




  
  9
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  9
  10
  
    
import org.apache.commons.logging.Log;


  		




  10
  11
  
    
import org.apache.commons.logging.LogFactory;


  		




  11
  12
  
    
import org.apache.hadoop.conf.Configuration;


  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/blackboard/SubmitOozieJobAction.java
    
  





  4
  4
  
    
import java.util.Map.Entry;


  		




  5
  5
  
    
import java.util.Properties;


  		




  6
  6
  
    

  		




  
  7
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  7
  8
  
    
import org.apache.commons.logging.Log;


  		




  8
  9
  
    
import org.apache.commons.logging.LogFactory;


  		




  9
  10
  
    
import org.apache.hadoop.conf.Configuration;


  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/blackboard/SequenceFileFeeder.java
    
  





  4
  4
  
    

  		




  5
  5
  
    
import eu.dnetlib.enabling.resultset.client.ResultSetClient;


  		




  6
  6
  
    
import eu.dnetlib.rmi.common.ResultSet;


  		




  
  7
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  7
  8
  
    
import org.apache.commons.lang.StringUtils;


  		




  8
  9
  
    
import org.apache.commons.logging.Log;


  		




  9
  10
  
    
import org.apache.commons.logging.LogFactory;


  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/blackboard/ImportEprHdfsAction.java
    
  





  3
  3
  
    
import java.io.IOException;


  		




  4
  4
  
    

  		




  5
  5
  
    
import eu.dnetlib.rmi.common.ResultSet;


  		




  
  6
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  6
  7
  
    
import org.apache.commons.logging.Log;


  		




  7
  8
  
    
import org.apache.commons.logging.LogFactory;


  		




  8
  9
  
    
import org.springframework.beans.factory.annotation.Autowired;


  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/blackboard/AbstractSubmitAction.java
    
  





  4
  4
  
    
import java.util.List;


  		




  5
  5
  
    
import java.util.Map;


  		




  6
  6
  
    

  		




  
  7
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  7
  8
  
    
import org.apache.commons.lang.StringUtils;


  		




  8
  9
  
    
import org.apache.commons.logging.Log;


  		




  9
  10
  
    
import org.apache.commons.logging.LogFactory;


  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/blackboard/SubmitAdminJobAction.java
    
  





  10
  10
  
    
import com.google.common.collect.Lists;


  		




  11
  11
  
    
import eu.dnetlib.data.hadoop.HadoopJob;


  		




  12
  12
  
    
import eu.dnetlib.data.hadoop.mapreduce.MapreduceJobMonitor;


  		




  
  13
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  13
  14
  
    
import eu.dnetlib.rmi.data.hadoop.HadoopJobType.AdminJobType;


  		




  14
  15
  
    
import eu.dnetlib.rmi.data.hadoop.HadoopServiceException;


  		




  15
  16
  
    
import eu.dnetlib.data.hadoop.utils.JobProfile;


  		




  ......




  34
  35
  
    
	 */


  		




  35
  36
  
    
	private static final Log log = LogFactory.getLog(SubmitAdminJobAction.class); // NOPMD by marko on 11/24/08 5:02 PM


  		




  36
  37
  
    

  		




  37
  
  
    
	/*


  		




  
  38
  
    
	/**


  		




  38
  39
  
    
	 * (non-Javadoc)


  		




  39
  40
  
    
	 *


  		




  40
  41
  
    
	 * @see


  		




  41
  
  
    
	 * eu.dnetlib.data.hadoop.action.SubmitMapreduceJobAction#executeAsync(eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler,


  		




  
  42
  
    
	 * eu.dnetlib.data.hadoop..blackboard.SubmitMapreduceJobAction#executeAsync(eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler,


  		




  42
  43
  
    
	 * eu.dnetlib.enabling.tools.blackboard.BlackboardJob)


  		




  43
  44
  
    
	 */


  		




  44
  45
  
    
	@Override


  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/blackboard/CreateHdfsDirectoryAction.java
    
  





  1
  1
  
    
package eu.dnetlib.data.hadoop.blackboard;


  		




  2
  2
  
    

  		




  3
  3
  
    
import eu.dnetlib.data.hadoop.HadoopServiceCore;


  		




  
  4
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  4
  5
  
    
import eu.dnetlib.rmi.data.hadoop.HadoopServiceException;


  		




  5
  6
  
    
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;


  		




  6
  7
  
    
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;


  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/blackboard/AbstractHadoopAction.java
    
  





  9
  9
  
    

  		




  10
  10
  
    
import javax.annotation.Resource;


  		




  11
  11
  
    

  		




  
  12
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  12
  13
  
    
import org.apache.commons.lang.exception.ExceptionUtils;


  		




  13
  14
  
    
import org.apache.commons.logging.Log;


  		




  14
  15
  
    
import org.apache.commons.logging.LogFactory;


  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/HadoopJob.java
    
  





  4
  4
  
    
import java.util.concurrent.Executor;


  		




  5
  5
  
    
import java.util.concurrent.Executors;


  		




  6
  6
  
    

  		




  
  7
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  7
  8
  
    
import org.apache.commons.logging.Log;


  		




  8
  9
  
    
import org.apache.commons.logging.LogFactory;


  		




  9
  10
  
    

  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/mapreduce/JobClientFactory.java
    
  





  2
  2
  
    

  		




  3
  3
  
    
import java.io.IOException;


  		




  4
  4
  
    

  		




  
  5
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  5
  6
  
    
import org.apache.hadoop.mapred.JobClient;


  		




  6
  7
  
    
import org.springframework.beans.factory.annotation.Autowired;


  		




  7
  8
  
    
import org.springframework.context.annotation.Lazy;


  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/mapreduce/HadoopJob.java
    
  





  2
  2
  
    

  		




  3
  3
  
    
import java.util.Properties;


  		




  4
  4
  
    

  		




  
  5
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  5
  6
  
    
import org.apache.hadoop.mapreduce.Job;


  		




  6
  7
  
    

  		




  7
  8
  
    
/**


  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/oozie/OozieClientFactory.java
    
  





  1
  1
  
    
package eu.dnetlib.data.hadoop.oozie;


  		




  2
  2
  
    

  		




  
  3
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  3
  4
  
    
import org.apache.commons.logging.Log;


  		




  4
  5
  
    
import org.apache.commons.logging.LogFactory;


  		




  5
  6
  
    
import org.apache.oozie.client.OozieClient;


  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/HadoopClientMap.java
    
  





  5
  5
  
    
import java.util.concurrent.Executors;


  		




  6
  6
  
    
import java.util.concurrent.TimeUnit;


  		




  7
  7
  
    

  		




  
  8
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  8
  9
  
    
import org.apache.commons.logging.Log;


  		




  9
  10
  
    
import org.apache.commons.logging.LogFactory;


  		




  10
  11
  
    
import org.apache.hadoop.hbase.client.HBaseAdmin;


  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/java/eu/dnetlib/data/hadoop/HadoopServiceImpl.java
    
  





  11
  11
  
    

  		




  12
  12
  
    
import eu.dnetlib.enabling.tools.AbstractBaseService;


  		




  13
  13
  
    
import eu.dnetlib.enabling.tools.blackboard.NotificationHandler;


  		




  
  14
  
    
import eu.dnetlib.rmi.data.hadoop.ClusterName;


  		




  14
  15
  
    
import eu.dnetlib.rmi.data.hadoop.HadoopJobDescriptor;


  		




  15
  16
  
    
import eu.dnetlib.rmi.data.hadoop.HadoopService;


  		




  16
  17
  
    
import eu.dnetlib.rmi.data.hadoop.HadoopServiceException;


  		












  

    
      modules/dnet-hadoop-services/trunk/src/main/resources/eu/dnetlib/data/actionmanager/applicationContext-dnet-actionmanager-service.xml
    
  





  47
  47
  
    

  		




  48
  48
  
    
	</bean>


  		




  49
  49
  
    

  		




  
  50
  
    
	<!--


  		




  50
  51
  
    
	<bean id="actionFactory" class="eu.dnetlib.data.actionmanager.actions.ActionFactory">


  		




  51
  52
  
    
		<property name="xslts">


  		




  52
  53
  
    
			<util:map>


  		




  ......




  63
  64
  
    
			</util:map>


  		




  64
  65
  
    
		</property>


  		




  65
  66
  
    
	</bean>


  		




  
  67
  
    
	-->


  		




  66
  68
  
    

  		




  67
  69
  
    
	<!-- Information Service Client -->


  		




  68
  70
  
    
	<bean id="actionmanagerISClient" class="eu.dnetlib.data.actionmanager.is.ISClient"


  		












Also available in:  Unified diff