Revision 29141
Added by Eri Katsari over 10 years ago
| modules/dnet-openaire-stats/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/statsExport/drivers/SqoopDriver.java | ||
|---|---|---|
| 64 | 64 |
for (Entry<String, String> table : tables.entrySet()) {
|
| 65 | 65 |
|
| 66 | 66 |
log.info("Importing " + table);
|
| 67 |
|
|
| 68 |
String[] str = { "export", "-Dsqoop.export.records.per.statement =" + RecsPerStatement, "-Dsqoop.export.statements.per.transaction = " + StatementPerTrans,
|
|
| 67 | 69 |
|
| 70 |
"--connect", connectionUrl, "--table", table.getKey(), |
|
| 71 |
|
|
| 72 |
"--export-dir", table.getValue(), |
|
| 73 |
// TODO : important!! this is necessary since MR multiple |
|
| 74 |
// outputs writer appends a \t char after writing the KEY |
|
| 75 |
// field in HDFS. |
|
| 76 |
// use this so sqoop can ignore it |
|
| 77 |
"--optionally-enclosed-by", " ", "--input-fields-terminated-by", delim, |
|
| 78 |
|
|
| 79 |
"--verbose", "--username", dbUser, "--password", |
|
| 80 |
dbPass, "--driver", "org.postgresql.Driver", "--batch", |
|
| 81 |
"--mapreduce-job-name", "Sqoop Stats Import", "-m", sqoopReducersCount }; |
|
| 82 |
|
|
| 83 |
|
|
| 84 |
// int ret = Sqoop.runTool(str); |
|
| 85 |
// if (ret != 0) {
|
|
| 86 |
// log.error("Could not run Sqoop Tool " + Integer.toString(ret));
|
|
| 87 |
// throw new RuntimeException("Could not run Sqoop Tool " +
|
|
| 88 |
// Integer.toString(ret)); |
|
| 89 |
// } |
|
| 68 | 90 |
SqoopTool tool = SqoopTool.getTool("export");
|
| 69 | 91 |
if (null == tool) {
|
| 70 | 92 |
log.error("No such sqoop tool: " + "dnetSqoop" + ". See 'sqoop help'.");
|
| ... | ... | |
| 118 | 140 |
ExportTool exportTool= new ExportTool(); |
| 119 | 141 |
|
| 120 | 142 |
Sqoop sqoop = new Sqoop( (com.cloudera.sqoop.tool.SqoopTool) tool, conf,opts); |
| 121 |
Sqoop.runSqoop(sqoop, args.toArray(new String[0]));
|
|
| 143 |
Sqoop.runSqoop(sqoop, str);
|
|
| 122 | 144 |
cleanUp(table.getKey()); |
| 123 | 145 |
} |
| 124 | 146 |
} |
Also available in: Unified diff