(path, reuseKeyValueInstances, conf);
+ } catch (IOException ioe) {
+ throw new IllegalStateException(path.toString(), ioe);
+ }
+ }
+
+}
Index: modules/dnet-hadoop-commons/tags/dnet-hadoop-commons-2.0.1/src/main/java/eu/dnetlib/data/hadoop/hdfs/SequenceFileIterator.java
===================================================================
--- modules/dnet-hadoop-commons/tags/dnet-hadoop-commons-2.0.1/src/main/java/eu/dnetlib/data/hadoop/hdfs/SequenceFileIterator.java (nonexistent)
+++ modules/dnet-hadoop-commons/tags/dnet-hadoop-commons-2.0.1/src/main/java/eu/dnetlib/data/hadoop/hdfs/SequenceFileIterator.java (revision 47956)
@@ -0,0 +1,99 @@
+package eu.dnetlib.data.hadoop.hdfs;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.ReflectionUtils;
+
+import com.google.common.collect.AbstractIterator;
+import com.google.common.io.Closeables;
+
+import eu.dnetlib.miscutils.collections.Pair;
+
+/**
+ *
+ * {@link java.util.Iterator} over a {@link SequenceFile}'s keys and values, as a {@link Pair} containing key and value.
+ *
+ */
+public final class SequenceFileIterator extends AbstractIterator> implements Closeable {
+
+ private final SequenceFile.Reader reader;
+ private final Configuration conf;
+ private final Class keyClass;
+ private final Class valueClass;
+ private final boolean noValue;
+ private K key;
+ private V value;
+ private final boolean reuseKeyValueInstances;
+
+ /**
+ * @throws IOException
+ * if path can't be read, or its key or value class can't be instantiated
+ */
+ @SuppressWarnings("unchecked")
+ public SequenceFileIterator(Path path, boolean reuseKeyValueInstances, Configuration conf) throws IOException {
+ key = null;
+ value = null;
+ FileSystem fs = path.getFileSystem(conf);
+ path = path.makeQualified(fs);
+ reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(path));
+ this.conf = conf;
+ keyClass = (Class) reader.getKeyClass();
+ valueClass = (Class) reader.getValueClass();
+ noValue = NullWritable.class.equals(valueClass);
+ this.reuseKeyValueInstances = reuseKeyValueInstances;
+ }
+
+ public Class getKeyClass() {
+ return keyClass;
+ }
+
+ public Class getValueClass() {
+ return valueClass;
+ }
+
+ @Override
+ public void close() {
+ key = null;
+ value = null;
+ try {
+ Closeables.close(reader, false);
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ endOfData();
+ }
+
+ @Override
+ protected Pair computeNext() {
+ if (!reuseKeyValueInstances || value == null) {
+ key = ReflectionUtils.newInstance(keyClass, conf);
+ if (!noValue) {
+ value = ReflectionUtils.newInstance(valueClass, conf);
+ }
+ }
+ try {
+ boolean available;
+ if (noValue) {
+ available = reader.next(key);
+ } else {
+ available = reader.next(key, value);
+ }
+ if (!available) {
+ close();
+ return null;
+ }
+ return new Pair(key, value);
+ } catch (IOException ioe) {
+ close();
+ throw new IllegalStateException(ioe);
+ }
+ }
+
+}
\ No newline at end of file
Index: modules/dnet-hadoop-commons/tags/dnet-hadoop-commons-2.0.1/src/main/java/eu/dnetlib/data/hadoop/config/ClusterName.java
===================================================================
--- modules/dnet-hadoop-commons/tags/dnet-hadoop-commons-2.0.1/src/main/java/eu/dnetlib/data/hadoop/config/ClusterName.java (nonexistent)
+++ modules/dnet-hadoop-commons/tags/dnet-hadoop-commons-2.0.1/src/main/java/eu/dnetlib/data/hadoop/config/ClusterName.java (revision 47956)
@@ -0,0 +1,34 @@
+package eu.dnetlib.data.hadoop.config;
+
+import java.util.Arrays;
+import java.util.List;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+
+/**
+ * Enumeration of the managed clusters.
+ *
+ * @author claudio
+ *
+ */
+public enum ClusterName {
+ DM, // Data Management
+ IIS; // Information Inference Service(s)
+
+ public static List asStringList() {
+ return Lists.newArrayList(Iterables.transform(asList(), new Function() {
+
+ @Override
+ public String apply(final ClusterName clusterName) {
+ return clusterName.toString();
+ }
+ }));
+ }
+
+ public static List asList() {
+ return Arrays.asList(ClusterName.values());
+ }
+
+}
Property changes on: modules/dnet-hadoop-commons/tags/dnet-hadoop-commons-2.0.1/src/main/java/eu/dnetlib/data/hadoop/config/ClusterName.java
___________________________________________________________________
Added: svn:mime-type
## -0,0 +1 ##
+text/plain
\ No newline at end of property
Index: modules/dnet-hadoop-commons/tags/dnet-hadoop-commons-2.0.1/src/main/java/eu/dnetlib/data/hadoop/config/ConfigurationFactory.java
===================================================================
--- modules/dnet-hadoop-commons/tags/dnet-hadoop-commons-2.0.1/src/main/java/eu/dnetlib/data/hadoop/config/ConfigurationFactory.java (nonexistent)
+++ modules/dnet-hadoop-commons/tags/dnet-hadoop-commons-2.0.1/src/main/java/eu/dnetlib/data/hadoop/config/ConfigurationFactory.java (revision 47956)
@@ -0,0 +1,73 @@
+package eu.dnetlib.data.hadoop.config;
+
+import java.io.IOException;
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.springframework.beans.factory.FactoryBean;
+import org.springframework.beans.factory.annotation.Required;
+import org.springframework.core.io.Resource;
+
+/**
+ * Factory bean for hadoop cluster configuration object
+ *
+ * @author claudio
+ *
+ */
+public class ConfigurationFactory implements FactoryBean {
+
+ private static final Log log = LogFactory.getLog(ConfigurationFactory.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ private Resource defaults;
+
+ public Configuration getConfiguration() {
+ try {
+ return getObject();
+ } catch (Exception e) {
+ throw new IllegalStateException("Unable to load hadoop configuration", e);
+ }
+ }
+
+ @Override
+ public Configuration getObject() throws Exception {
+ return initConfiguration(defaultProperties());
+ }
+
+ @Override
+ public Class> getObjectType() {
+ return Configuration.class;
+ }
+
+ @Override
+ public boolean isSingleton() {
+ return true;
+ }
+
+ private Configuration initConfiguration(final Properties p) {
+ final Configuration conf = new Configuration(false);
+ for (Entry