| 1 |
28111
|
eri.katsar
|
package eu.dnetlib.data.mapreduce.hbase.statsExport.drivers;
|
| 2 |
|
|
|
| 3 |
|
|
import java.io.InputStream;
|
| 4 |
|
|
import java.util.Properties;
|
| 5 |
|
|
|
| 6 |
|
|
import org.apache.hadoop.conf.Configuration;
|
| 7 |
|
|
import org.apache.hadoop.mapreduce.Job;
|
| 8 |
|
|
import org.apache.log4j.Logger;
|
| 9 |
|
|
|
| 10 |
|
|
import eu.dnetlib.data.mapreduce.hbase.statsExport.StatsJob;
|
| 11 |
|
|
|
| 12 |
|
|
/**
|
| 13 |
|
|
* @author eri
|
| 14 |
|
|
*
|
| 15 |
|
|
*/
|
| 16 |
|
|
public class StatsJobDriver {
|
| 17 |
|
|
|
| 18 |
|
|
private Logger log = Logger.getLogger(this.getClass());
|
| 19 |
28203
|
eri.katsar
|
|
| 20 |
28111
|
eri.katsar
|
private Configuration configuration = new Configuration();
|
| 21 |
|
|
|
| 22 |
|
|
private StatsJob statsJob;
|
| 23 |
|
|
|
| 24 |
|
|
private Job job;
|
| 25 |
|
|
|
| 26 |
|
|
private Properties props;
|
| 27 |
|
|
|
| 28 |
28138
|
eri.katsar
|
public void init() throws Exception {
|
| 29 |
28111
|
eri.katsar
|
InputStream file = ClassLoader.getSystemResourceAsStream("eu/dnetlib/data/mapreduce/hbase/statsExport/StatsProperties");
|
| 30 |
|
|
props = new Properties();
|
| 31 |
|
|
props.load(file);
|
| 32 |
|
|
file.close();
|
| 33 |
28511
|
eri.katsar
|
configuration.set("stats.delim", props.getProperty("Stats.delimCharacter"));
|
| 34 |
28111
|
eri.katsar
|
|
| 35 |
28138
|
eri.katsar
|
configuration.set("hbase.mapreduce.inputtable", props.getProperty("Stats.HbaseSourceTable"));
|
| 36 |
28111
|
eri.katsar
|
|
| 37 |
28138
|
eri.katsar
|
if (!props.getProperty("Stats.outputPath").endsWith("/")) {
|
| 38 |
|
|
props.setProperty("Stats.outputPath", props.getProperty("Stats.outputPath") + "/");
|
| 39 |
28111
|
eri.katsar
|
}
|
| 40 |
|
|
|
| 41 |
28138
|
eri.katsar
|
configuration.set("mapred.output.dir", props.getProperty("Stats.outputPath"));
|
| 42 |
28111
|
eri.katsar
|
|
| 43 |
28511
|
eri.katsar
|
configuration.set("stats.nullNum", props.getProperty("Stats.nullNumericField"));
|
| 44 |
|
|
configuration.set("stats.nullString", props.getProperty("Stats.nullStringField"));
|
| 45 |
28203
|
eri.katsar
|
|
| 46 |
28111
|
eri.katsar
|
job = new Job(configuration);
|
| 47 |
|
|
|
| 48 |
|
|
statsJob = new StatsJob();
|
| 49 |
28138
|
eri.katsar
|
statsJob.setJobDetails(job, props);
|
| 50 |
28111
|
eri.katsar
|
|
| 51 |
|
|
}
|
| 52 |
|
|
|
| 53 |
|
|
public void run() throws Exception {
|
| 54 |
|
|
|
| 55 |
|
|
long startTime = System.currentTimeMillis();
|
| 56 |
|
|
long endtime;
|
| 57 |
|
|
double totalTime1;
|
| 58 |
|
|
|
| 59 |
|
|
// TODO here, start the Stats Export Job: Read from HBASE, and dump to
|
| 60 |
|
|
// HDFS
|
| 61 |
|
|
boolean completionStatus = job.waitForCompletion(true);
|
| 62 |
|
|
|
| 63 |
|
|
if (completionStatus) {
|
| 64 |
|
|
endtime = System.currentTimeMillis();
|
| 65 |
|
|
totalTime1 = (endtime - startTime) / 60000;
|
| 66 |
|
|
|
| 67 |
|
|
log.info("Time taken for MR Export : " + totalTime1 + " minutes");
|
| 68 |
|
|
|
| 69 |
|
|
} else {
|
| 70 |
|
|
log.error("JOB FAILED :");
|
| 71 |
|
|
|
| 72 |
|
|
}
|
| 73 |
|
|
|
| 74 |
|
|
}
|
| 75 |
|
|
|
| 76 |
|
|
}
|