Revision 28495
Added by Eri Katsari about 10 years ago
| modules/dnet-openaire-stats/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/statsExport/drivers/SqoopDriver.java | ||
|---|---|---|
| 40 | 40 |
private String sqoopReducersCount; |
| 41 | 41 |
private FileSystem hdfs; |
| 42 | 42 |
|
| 43 |
private boolean useHdfsStore = false; |
|
| 44 |
|
|
| 43 | 45 |
/** |
| 44 | 46 |
* Driver for the Sqoop tool. Calls the Sqoop Client for each <input file, |
| 45 | 47 |
* destination table> pair given in the @tables argument. |
| ... | ... | |
| 81 | 83 |
if (tableMappings == null || tableMappings.isEmpty()) {
|
| 82 | 84 |
throw new Exception("Could not load Table Mappings in sqoop init job");
|
| 83 | 85 |
} |
| 84 |
hdfs = FileSystem.get(new Configuration()); |
|
| 85 |
RemoteIterator<LocatedFileStatus> files; |
|
| 86 |
|
|
| 87 |
|
|
| 88 |
Path exportPath=new Path(hdfs.getUri()+outputPath); |
|
| 89 |
try {
|
|
| 90 |
files = hdfs.listFiles(exportPath, false); |
|
| 86 |
ArrayList<String> fileNames; |
|
| 87 |
if (useHdfsStore) {
|
|
| 88 |
fileNames = listHdfsDir(); |
|
| 89 |
} else {
|
|
| 90 |
fileNames = listFilesystemDir(); |
|
| 91 |
} |
|
| 91 | 92 |
|
| 92 |
} catch (FileNotFoundException e1) {
|
|
| 93 |
log.error("HDFS file path with exported data does not exist : " + outputPath);
|
|
| 94 |
throw new Exception("HDFS file path with exported data does not exist : " + outputPath, e1);
|
|
| 95 |
|
|
| 96 |
} catch (IOException e1) {
|
|
| 97 |
log.error("HDFS file path with exported data does not exist :" + outputPath);
|
|
| 98 |
throw new Exception("HDFS file path with exported data does not exist " + outputPath, e1);
|
|
| 99 |
|
|
| 100 |
} |
|
| 101 | 93 |
HashMap<String, String> tables = new HashMap<String, String>(); |
| 102 |
ArrayList<String> fileNames = new ArrayList<String>(); |
|
| 103 | 94 |
|
| 104 |
while (files.hasNext()) {
|
|
| 105 |
|
|
| 106 |
String fileName = files.next().getPath().toString(); |
|
| 107 |
|
|
| 108 |
log.info(fileName); |
|
| 109 |
fileNames.add(fileName); |
|
| 110 |
|
|
| 111 |
} |
|
| 112 | 95 |
// Table mappings containt the mapping between HDFS files and the Stats |
| 113 | 96 |
// DB table that each should be imported to |
| 114 | 97 |
for (Entry<Object, Object> e : tableMappings.entrySet()) {
|
| ... | ... | |
| 142 | 125 |
|
| 143 | 126 |
} |
| 144 | 127 |
|
| 128 |
private ArrayList<String> listHdfsDir() throws Exception {
|
|
| 129 |
hdfs = FileSystem.get(new Configuration()); |
|
| 130 |
RemoteIterator<LocatedFileStatus> files; |
|
| 131 |
|
|
| 132 |
Path exportPath = new Path(hdfs.getUri() + outputPath); |
|
| 133 |
try {
|
|
| 134 |
files = hdfs.listFiles(exportPath, false); |
|
| 135 |
|
|
| 136 |
} catch (FileNotFoundException e1) {
|
|
| 137 |
log.error("HDFS file path with exported data does not exist : " + outputPath);
|
|
| 138 |
throw new Exception("HDFS file path with exported data does not exist : " + outputPath, e1);
|
|
| 139 |
|
|
| 140 |
} catch (IOException e1) {
|
|
| 141 |
log.error("HDFS file path with exported data does not exist :" + outputPath);
|
|
| 142 |
throw new Exception("HDFS file path with exported data does not exist " + outputPath, e1);
|
|
| 143 |
|
|
| 144 |
} |
|
| 145 |
ArrayList<String> fileNames = new ArrayList<String>(); |
|
| 146 |
|
|
| 147 |
while (files.hasNext()) {
|
|
| 148 |
|
|
| 149 |
String fileName = files.next().getPath().toString(); |
|
| 150 |
|
|
| 151 |
log.info(fileName); |
|
| 152 |
fileNames.add(fileName); |
|
| 153 |
|
|
| 154 |
} |
|
| 155 |
return fileNames; |
|
| 156 |
} |
|
| 157 |
|
|
| 158 |
private ArrayList<String> listFilesystemDir() throws Exception {
|
|
| 159 |
ArrayList<String> fileNames = new ArrayList<String>(); |
|
| 160 |
|
|
| 161 |
String files; |
|
| 162 |
File folder = new File(outputPath); |
|
| 163 |
File[] listOfFiles = folder.listFiles(); |
|
| 164 |
|
|
| 165 |
for (int i = 0; i < listOfFiles.length; i++) {
|
|
| 166 |
|
|
| 167 |
if (listOfFiles[i].isFile()) {
|
|
| 168 |
files = listOfFiles[i].getName(); |
|
| 169 |
log.info(files); |
|
| 170 |
fileNames.add(files); |
|
| 171 |
System.out.println(files); |
|
| 172 |
} |
|
| 173 |
} |
|
| 174 |
|
|
| 175 |
return fileNames; |
|
| 176 |
} |
|
| 177 |
|
|
| 145 | 178 |
/** |
| 146 | 179 |
* Cleans up auto-generated Sqoop class files |
| 147 | 180 |
* |
| ... | ... | |
| 278 | 311 |
this.hdfs = hdfs; |
| 279 | 312 |
} |
| 280 | 313 |
|
| 314 |
public boolean isUseHdfsStore() {
|
|
| 315 |
return useHdfsStore; |
|
| 316 |
} |
|
| 317 |
|
|
| 318 |
public void setUseHdfsStore(boolean useHdfsStore) {
|
|
| 319 |
this.useHdfsStore = useHdfsStore; |
|
| 320 |
} |
|
| 321 |
|
|
| 281 | 322 |
} |
Also available in: Unified diff