Revision 28584
Added by Eri Katsari about 10 years ago
| modules/dnet-openaire-stats/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/statsExport/drivers/SqoopDriver.java | ||
|---|---|---|
| 21 | 21 |
private String TABLE_MAP_PATH = "eu/dnetlib/data/mapreduce/hbase/statsExport/exportTables"; |
| 22 | 22 |
|
| 23 | 23 |
private Properties tableMappings = new Properties(); |
| 24 |
|
|
| 24 |
|
|
| 25 | 25 |
private String outputPath; |
| 26 | 26 |
private String dbUser; |
| 27 | 27 |
private String dbPass; |
| ... | ... | |
| 124 | 124 |
|
| 125 | 125 |
private ArrayList<String> listHdfsDir() throws Exception {
|
| 126 | 126 |
hdfs = FileSystem.get(conf); |
| 127 |
RemoteIterator<LocatedFileStatus> files; |
|
| 128 | 127 |
|
| 129 |
Path exportPath = new Path(outputPath);
|
|
| 128 |
RemoteIterator<LocatedFileStatus> Files;
|
|
| 130 | 129 |
|
| 131 |
// Path exportPath = new Path(outputPath); |
|
| 132 | 130 |
try {
|
| 133 |
files = hdfs.listFiles(hdfs.resolvePath(exportPath), true); |
|
| 134 |
if (files == null) {
|
|
| 131 |
Path exportPath = new Path(hdfs.getUri() + outputPath); |
|
| 132 |
Files = hdfs.listFiles(exportPath, false); |
|
| 133 |
if (Files == null) {
|
|
| 135 | 134 |
log.error("No files generated in " + outputPath + " at " + exportPath.getName());
|
| 136 | 135 |
throw new Exception("No files generated in " + outputPath + " at " + exportPath.getName() + " in " + hdfs.resolvePath(exportPath));
|
| 137 | 136 |
|
| 138 | 137 |
} |
| 139 | 138 |
} catch (Exception e) {
|
| 140 |
log.error("HDFS file path with exported data does not exist : " + outputPath + " at " + exportPath.getName());
|
|
| 141 |
throw new Exception("HDFS file path with exported data does not exist : " + outputPath + " at " + exportPath.getName(), e);
|
|
| 139 |
log.error("HDFS file path with exported data does not exist : " + outputPath + " at " +new Path(hdfs.getUri() + outputPath));
|
|
| 140 |
throw new Exception("HDFS file path with exported data does not exist : " + outputPath + " at " + new Path(hdfs.getUri() + outputPath), e);
|
|
| 142 | 141 |
|
| 143 | 142 |
} |
| 144 | 143 |
ArrayList<String> fileNames = new ArrayList<String>(); |
| 145 | 144 |
|
| 146 |
while (files.hasNext()) {
|
|
| 145 |
while (Files.hasNext()) {
|
|
| 147 | 146 |
|
| 148 |
String fileName = files.next().getPath().toString();
|
|
| 147 |
String fileName = Files.next().getPath().toString();
|
|
| 149 | 148 |
|
| 150 | 149 |
log.info(fileName); |
| 151 | 150 |
fileNames.add(fileName); |
| ... | ... | |
| 284 | 283 |
this.tableMappings = tableMappings; |
| 285 | 284 |
} |
| 286 | 285 |
|
| 287 |
|
|
| 288 |
|
|
| 289 | 286 |
public String getOutputPath() {
|
| 290 | 287 |
return outputPath; |
| 291 | 288 |
} |
Also available in: Unified diff