Revision 35701
Added by Mateusz Kobos about 9 years ago
| DocumentToDataSetConfidenceLevelBasedCleaner.java | ||
|---|---|---|
| 4 | 4 |
import java.util.Map; |
| 5 | 5 |
|
| 6 | 6 |
import org.apache.avro.file.DataFileWriter; |
| 7 |
import org.apache.hadoop.conf.Configuration; |
|
| 7 | 8 |
import org.apache.hadoop.fs.FileSystem; |
| 8 | 9 |
import org.apache.hadoop.fs.Path; |
| 9 | 10 |
import org.apache.log4j.Logger; |
| 10 | 11 |
|
| 11 |
import eu.dnetlib.iis.core.java.HadoopContext; |
|
| 12 | 12 |
import eu.dnetlib.iis.core.java.PortBindings; |
| 13 | 13 |
import eu.dnetlib.iis.core.java.ProcessUtils; |
| 14 | 14 |
import eu.dnetlib.iis.core.java.io.CloseableIterator; |
| ... | ... | |
| 59 | 59 |
} |
| 60 | 60 |
|
| 61 | 61 |
@Override |
| 62 |
public void run(PortBindings portBindings, HadoopContext context,
|
|
| 62 |
public void run(PortBindings portBindings, Configuration conf,
|
|
| 63 | 63 |
Map<String, String> parameters) throws Exception {
|
| 64 | 64 |
String confidenceLevelThresholdStr = ProcessUtils.getParameterValue( |
| 65 | 65 |
CONFIDENCE_LEVEL_THRESHOLD, |
| 66 |
context.getConfiguration(), parameters);
|
|
| 66 |
conf, parameters);
|
|
| 67 | 67 |
if (confidenceLevelThresholdStr==null || confidenceLevelThresholdStr.isEmpty()) {
|
| 68 | 68 |
throw new RuntimeException("no confidence level threshold "
|
| 69 | 69 |
+ "parameter provided: '" + CONFIDENCE_LEVEL_THRESHOLD + "'"); |
| ... | ... | |
| 73 | 73 |
Map<String, Path> input = portBindings.getInput(); |
| 74 | 74 |
Map<String, Path> output = portBindings.getOutput(); |
| 75 | 75 |
|
| 76 |
FileSystem fs = FileSystem.get(context.getConfiguration());
|
|
| 76 |
FileSystem fs = FileSystem.get(conf);
|
|
| 77 | 77 |
|
| 78 | 78 |
CloseableIterator<DocumentToDataSet> it = DataStore.getReader( |
| 79 | 79 |
new FileSystemPath(fs, input.get(inputPort))); |
Also available in: Unified diff
Removing usage of working_dir from Java workflow node.