D-Net

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-hadoop-commons/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-hadoop-commons"}

package eu.dnetlib.data.hadoop.hdfs;

import java.io.IOException;

import java.util.Iterator;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.Writable;

import eu.dnetlib.miscutils.collections.Pair;

/**

 * <p>

 * {@link Iterable} counterpart to {@link SequenceFileIterator}.

 * </p>

 */

public final class SequenceFileIterable<K extends Writable, V extends Writable> implements Iterable<Pair<K, V>> {

	private final Path path;

	private final boolean reuseKeyValueInstances;

	private final Configuration conf;

	/**

	 * Like {@link #SequenceFileIterable(Path, boolean, Configuration)} but key and value instances are not reused by

	 * default.

	 * 

	 * @param path

	 *            file to iterate over

	 */

	public SequenceFileIterable(Path path, Configuration conf) {

		this(path, false, conf);

	}

	/**

	 * @param path

	 *            file to iterate over

	 * @param reuseKeyValueInstances

	 *            if true, reuses instances of the key and value object instead of creating a new one for each read from

	 *            the file

	 */

	public SequenceFileIterable(Path path, boolean reuseKeyValueInstances, Configuration conf) {

		this.path = path;

		this.reuseKeyValueInstances = reuseKeyValueInstances;

		this.conf = conf;

	}

	@Override

	public Iterator<Pair<K, V>> iterator() {

		try {

			return new SequenceFileIterator<K, V>(path, reuseKeyValueInstances, conf);

		} catch (IOException ioe) {

			throw new IllegalStateException(path.toString(), ioe);

		}

	}

}

package eu.dnetlib.data.hadoop.hdfs;

import java.io.Closeable;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.SequenceFile;

import org.apache.hadoop.io.Writable;

import org.apache.hadoop.util.ReflectionUtils;

import com.google.common.collect.AbstractIterator;

import com.google.common.io.Closeables;

import eu.dnetlib.miscutils.collections.Pair;

/**

 * <p>

 * {@link java.util.Iterator} over a {@link SequenceFile}'s keys and values, as a {@link Pair} containing key and value.

 * </p>

 */

public final class SequenceFileIterator<K extends Writable, V extends Writable> extends AbstractIterator<Pair<K, V>> implements Closeable {

	private final SequenceFile.Reader reader;

	private final Configuration conf;

	private final Class<K> keyClass;

	private final Class<V> valueClass;

	private final boolean noValue;

	private K key;

	private V value;

	private final boolean reuseKeyValueInstances;

	/**

	 * @throws IOException

	 *             if path can't be read, or its key or value class can't be instantiated

	 */

	@SuppressWarnings("unchecked")

	public SequenceFileIterator(Path path, boolean reuseKeyValueInstances, Configuration conf) throws IOException {

		key = null;

		value = null;

		FileSystem fs = path.getFileSystem(conf);

		path = path.makeQualified(fs);

		reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(path));

		this.conf = conf;

		keyClass = (Class<K>) reader.getKeyClass();

		valueClass = (Class<V>) reader.getValueClass();

		noValue = NullWritable.class.equals(valueClass);

		this.reuseKeyValueInstances = reuseKeyValueInstances;

	}

	public Class<K> getKeyClass() {

		return keyClass;

	}

	public Class<V> getValueClass() {

		return valueClass;

	}

	@Override

	public void close() {

		key = null;

		value = null;

		try {

			Closeables.close(reader, false);

		} catch (IOException e) {

			throw new IllegalStateException(e);

		}

		endOfData();

	}

	@Override

	protected Pair<K, V> computeNext() {

		if (!reuseKeyValueInstances || value == null) {

			key = ReflectionUtils.newInstance(keyClass, conf);

			if (!noValue) {

				value = ReflectionUtils.newInstance(valueClass, conf);

			}

		}

		try {

			boolean available;

			if (noValue) {

				available = reader.next(key);

			} else {

				available = reader.next(key, value);

			}

			if (!available) {

				close();

				return null;

			}

			return new Pair<K, V>(key, value);

		} catch (IOException ioe) {

			close();

			throw new IllegalStateException(ioe);

		}

	}

}

package eu.dnetlib.data.hadoop.hdfs;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileStatus;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.fs.PathFilter;

import org.apache.hadoop.io.Text;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import eu.dnetlib.miscutils.collections.Pair;

public class SequenceFileUtils {

	public static final int NO_LIMIT = -1;

	public static Iterable<Pair<Text, Text>> read(Path path, final Configuration conf) throws IOException {

		return Iterables.concat(Iterables.transform(new SequenceFileUtils().doRead(path, conf), new Function<Path, SequenceFileIterable<Text, Text>>() {

			@Override

			public SequenceFileIterable<Text, Text> apply(Path path) {

				return new SequenceFileIterable<Text, Text>(path, conf);

			}

		}));

	}

	public static Iterable<Pair<Text, Text>> read(Path path, final Configuration conf, final int readMax) throws IOException {

		if (readMax != NO_LIMIT)

			return Iterables.limit(read(path, conf), readMax);

		else

			return read(path, conf);

	}

	private final PathFilter f = new PathFilter() {

		@Override

		public boolean accept(Path path) {

			return path.getName().contains("part-r");

		}

	};

	private Iterable<Path> doRead(Path path, final Configuration conf) throws IOException {

		FileSystem fs = FileSystem.get(conf);

		return Iterables.transform(Lists.newArrayList(fs.listStatus(path, f)), new Function<FileStatus, Path>() {

			@Override

			public Path apply(FileStatus fs) {

				Path path = fs.getPath();

				System.out.println("downloading xml files from path: " + path.toString());

				return path;

			}

		});

	}

}

package eu.dnetlib.data.hadoop.hdfs;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.SequenceFile;

import org.apache.hadoop.io.SequenceFile.CompressionType;

import org.apache.hadoop.io.SequenceFile.Writer.Option;

import org.apache.hadoop.io.Writable;

import org.springframework.beans.factory.annotation.Required;

/**

 * Factory for SequenceFile.Writer instances

 *

 * @author claudio

 *

 */

public class SequenceFileWriterFactory {

	private Class<? extends Writable> keyClass;

	private Class<? extends Writable> valueClass;

	private String compressionType;

	private String blockSize = "64M";

	private short replication = 3;

	public SequenceFile.Writer getSequenceFileWriter(

			final Class<? extends Writable> keyClass,

			final Class<? extends Writable> valueClass,

			final Configuration conf,

			final Path sequenceFilePath) throws IOException {

		Configuration newConf = new Configuration(conf);

		newConf.set("dfs.blocksize", getBlockSize());

		Option oFile = SequenceFile.Writer.file(sequenceFilePath);

		Option oKey = SequenceFile.Writer.keyClass(keyClass);

		Option oValue = SequenceFile.Writer.valueClass(valueClass);

		Option oCmp = SequenceFile.Writer.compression(CompressionType.valueOf(getCompressionType()));

		Option oRpl = SequenceFile.Writer.replication(getReplication());

		return SequenceFile.createWriter(newConf, oFile, oKey, oValue, oCmp, oRpl);

	}

	public SequenceFile.Writer getSequenceFileWriter(final Configuration conf, final Path sequenceFilePath) throws IOException {

		return getSequenceFileWriter(getKeyClass(), getValueClass(), conf, sequenceFilePath);

	}

	public Class<? extends Writable> getKeyClass() {

		return keyClass;

	}

	@Required

	public void setKeyClass(final Class<? extends Writable> keyClass) {

		this.keyClass = keyClass;

	}

	public Class<? extends Writable> getValueClass() {

		return valueClass;

	}

	@Required

	public void setValueClass(final Class<? extends Writable> valueClass) {

		this.valueClass = valueClass;

	}

	public String getCompressionType() {

		return compressionType;

	}

	@Required

	public void setCompressionType(final String compressionType) {

		this.compressionType = compressionType;

	}

	public String getBlockSize() {

		return blockSize;

	}

	public void setBlockSize(final String blockSize) {

		this.blockSize = blockSize;

	}

	public short getReplication() {

		return replication;

	}

	public void setReplication(final short replication) {

		this.replication = replication;

	}

}

package eu.dnetlib.data.hadoop.config;

import java.io.IOException;

import java.util.Map.Entry;

import java.util.Properties;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.hadoop.conf.Configuration;

import org.springframework.beans.factory.FactoryBean;

import org.springframework.beans.factory.annotation.Required;

import org.springframework.core.io.Resource;

/**

 * Factory bean for hadoop cluster configuration object

 * 

 * @author claudio

 * 

 */

public class ConfigurationFactory implements FactoryBean<Configuration> {

	private static final Log log = LogFactory.getLog(ConfigurationFactory.class); // NOPMD by marko on 11/24/08 5:02 PM

	private Resource defaults;

	public Configuration getConfiguration() {

		try {

			return getObject();

		} catch (Exception e) {

			throw new IllegalStateException("Unable to load hadoop configuration", e);

		}

	}

	@Override

	public Configuration getObject() throws Exception {

		return initConfiguration(defaultProperties());

	}

	@Override

	public Class<?> getObjectType() {

		return Configuration.class;

	}

	@Override

	public boolean isSingleton() {

		return true;

	}

	private Configuration initConfiguration(final Properties p) {

		final Configuration conf = new Configuration(false);

		for (Entry<Object, Object> e : p.entrySet()) {

			conf.set(e.getKey().toString(), e.getValue().toString());

			log.info(e.getKey().toString() + ": " + conf.get(e.getKey().toString()));

		}

		return conf;

	}

	private Properties defaultProperties() throws IOException {

		Properties p = new Properties();

		p.load(defaults.getInputStream());

		return p;

	}

	public Resource getDefaults() {

		return defaults;

	}

	@Required

	public void setDefaults(Resource defaults) {

		this.defaults = defaults;

	}

}

package eu.dnetlib.data.hadoop.config;

import java.util.Arrays;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

/**

 * Enumeration of the managed clusters.

 *

 * @author claudio

 *

 */

public enum ClusterName {

	DM, // Data Management

	IIS; // Information Inference Service(s)

	public static List<String> asStringList() {

		return Lists.newArrayList(Iterables.transform(asList(), new Function<ClusterName, String>() {

			@Override

			public String apply(final ClusterName clusterName) {

				return clusterName.toString();

			}

		}));

	}

	public static List<ClusterName> asList() {

		return Arrays.asList(ClusterName.values());

	}

}

package eu.dnetlib.data.hadoop.config;

import java.util.Map;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.hadoop.conf.Configuration;

import org.springframework.beans.BeansException;

import org.springframework.beans.factory.BeanFactory;

import org.springframework.beans.factory.BeanFactoryAware;

import org.springframework.beans.factory.ListableBeanFactory;

import org.springframework.beans.factory.NoSuchBeanDefinitionException;

import com.google.common.base.Function;

import com.google.common.collect.Maps;

public class ConfigurationEnumerator implements BeanFactoryAware {

	private static final Log log = LogFactory.getLog(ConfigurationEnumerator.class); // NOPMD by marko on 11/24/08 5:02 PM

	/**

	 * bean factory.

	 */

	private ListableBeanFactory beanFactory;

	private final Function<ConfigurationFactory, Configuration> asConfiguration = new Function<ConfigurationFactory, Configuration>() {

		@Override

		public Configuration apply(ConfigurationFactory factory) {

			return factory.getConfiguration();

		}

	};

	/**

	 * Get all beans implementing the ConfiguredJob interface.

	 * 

	 * @return

	 */

	public Map<String, Configuration> getAll() {

		return Maps.transformValues(beanFactory.getBeansOfType(ConfigurationFactory.class), asConfiguration);

	}

	@Override

	public void setBeanFactory(final BeanFactory beanFactory) throws BeansException {

		this.beanFactory = (ListableBeanFactory) beanFactory;

	}

	public ListableBeanFactory getBeanFactory() {

		return beanFactory;

	}

	/**

	 * Get given Configuration or null.

	 * 

	 * @param name

	 * @return

	 */

	public Configuration get(final ClusterName name) {

		try {

			return (beanFactory.getBean(name.toString(), Configuration.class));

		} catch (final NoSuchBeanDefinitionException e) {

			log.error("undefined bean: " + name, e);

			return null;

		}

	}

}

package eu.dnetlib.data.transform;

public enum ColumnType {

	base64,

	text,

}

package eu.dnetlib.data.transform;

import java.io.StringReader;

import java.util.List;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerException;

import org.apache.commons.codec.binary.Base64;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.Element;

import org.dom4j.Node;

import org.dom4j.io.DocumentResult;

import org.dom4j.io.DocumentSource;

import org.dom4j.io.SAXReader;

import com.google.common.base.Function;

import com.google.common.collect.Lists;

public class XsltRowTransformer implements Function<String, List<Row>> {

	private static final Log log = LogFactory.getLog(XsltRowTransformer.class); // NOPMD by marko on 11/24/08 5:02 PM

	private Transformer transformer;

	private SAXReader reader = new SAXReader();

	public XsltRowTransformer(final Transformer transformer) {

		this.transformer = transformer;

		log.info(String.format("using trasformer: '%s'", getTransformerClassName()));

	}

	private Document transform(final Document doc) {

		if (transformer == null)

			return doc;

		final DocumentResult result = new DocumentResult();

		try {

			transformer.transform(new DocumentSource(doc), result);

			return result.getDocument();

		} catch (final TransformerException e) {

			throw new RuntimeException("Unable to transform document:\n" + doc.asXML(), e);

		}

	}

	private Document transform(final String xml) {

		try {

			return transform(reader.read(new StringReader(xml)));

		} catch (final DocumentException e) {

			log.error("Error parsing xml:\n" + xml, e);

			throw new RuntimeException("Unable to parse document:\n" + xml, e);

		}

	}

	private List<Row> transformAsListRow(final String xml) {

		final Document doc = transform(xml);

		final List<Row> rows = Lists.newArrayList();

		if (doc == null)

			return rows;

		for (final Object or : doc.selectNodes("//ROW")) {

			final Element row = (Element) or;

			final String columnFamily = row.valueOf("@columnFamily");

			final String key = row.valueOf("@key");

			if ((key == null) || key.isEmpty())

				throw new RuntimeException("Attribute 'key' is missing in XSLT");

			if ((columnFamily == null) || columnFamily.isEmpty())

				throw new RuntimeException("Attribute 'columnFamily' is missing in XSLT");

			final List<Column<String, byte[]>> cols = Lists.newArrayList();

			for (final Object of : row.selectNodes("./QUALIFIER")) {

				final Node node = (Node) of;

				final String name = node.valueOf("@name");

				final String type = node.valueOf("@type");

				final byte[] value = decode(node.getText().trim(), type);

				cols.add(new Column<String, byte[]>(name, value));

			}

			rows.add(new Row(columnFamily, key, cols));

		}

		return rows;

	}

	public String getTransformerClassName() {

		return transformer != null ? transformer.getClass().getName() : "null";

	}

	private byte[] decode(final String value, final String type) {

		if ("base64".equals(type))

			return Base64.decodeBase64(value);

		return value.getBytes();

	}

	@Override

	public List<Row> apply(final String xml) {

		return transformAsListRow(xml);

	}

}

package eu.dnetlib.data.transform;

import java.util.Collection;

import java.util.Iterator;

import java.util.List;

import java.util.Map;

import com.google.common.collect.Maps;

public class Row implements Iterable<Column<String, byte[]>> {

	private String columnFamily;

	private String key;

	private Map<String, Column<String, byte[]>> columns;

	public Row(final Row row) {

		this.columnFamily = row.getColumnFamily();

		this.key = row.getKey();

		this.columns = row.columns;

	}

	public Row(final String columnFamily, final String key) {

		this.columnFamily = columnFamily;

		this.key = key;

		this.columns = Maps.newHashMap();

	}

	public Row(final String columnFamily, final String key, final List<Column<String, byte[]>> columns) {

		this(columnFamily, key);

		this.setColumns(columns);

	}

	public void addColumn(final Column<String, byte[]> column) {

		getColumns().add(column);

	}

	public String getKey() {

		return key;

	}

	public Collection<Column<String, byte[]>> getColumns() {

		return columns.values();

	}

	public Column<String, byte[]> getColumn(final String name) {

		return columns.get(name);

	}

	public void setColumn(final String name, final Column<String, byte[]> newColumn) {

		columns.put(name, newColumn);

	}

	public void setColumns(final List<Column<String, byte[]>> columns) {

		this.columns.clear();

		for (final Column<String, byte[]> col : columns) {

			this.columns.put(col.getName(), col);

		}

	}

	public String getColumnFamily() {

		return columnFamily;

	}

	@Override

	public Iterator<Column<String, byte[]>> iterator() {

		return getColumns().iterator();

	}

	@Override

	public String toString() {

		return "\nRow [ KEY: " + key + ",\n\tCF: " + columnFamily + "\n\tCOLS: {" + columns + "}]";

	}

}

package eu.dnetlib.data.transform;

public class Column<N, V> {

	private N name;

	private V value;

	public static <N, V> Column<N, V> newInstance(final N name, final V value) {

		return new Column<N, V>(name, value);

	}

	public Column(final N name, final V value) {

		this.name = name;

		this.value = value;

	}

	public N getName() {

		return name;

	}

	public V getValue() {

		return value;

	}

}

package eu.dnetlib.data.transform;

import java.io.StringReader;

import java.util.Map;

import java.util.Map.Entry;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerFactory;

import org.apache.commons.collections.MapUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.io.DocumentSource;

import org.dom4j.io.SAXReader;

public class XsltRowTransformerFactory {

	private static final Log log = LogFactory.getLog(XsltRowTransformerFactory.class); // NOPMD by marko on 11/24/08 5:02 PM

	public XsltRowTransformer getTransformer(final String xslt) {

		return doGetTransformer(xslt, null);

	}

	public XsltRowTransformer getTransformer(final String xslt, final Map<String, Object> params) {

		return doGetTransformer(xslt, params);

	}

	private XsltRowTransformer doGetTransformer(final String xslt, final Map<String, Object> params) {

		try {

			if ((xslt == null) || xslt.isEmpty()) return new XsltRowTransformer(null);

			final TransformerFactory factory = TransformerFactory.newInstance();

			if (log.isDebugEnabled()) {

				log.debug("using transformer factory: '" + factory.getClass().getCanonicalName() + "'");

			}

			final Transformer t = factory.newTransformer(new DocumentSource(new SAXReader().read(new StringReader(xslt))));

			if (!MapUtils.isEmpty(params)) {

				for (final Entry<String, Object> e : params.entrySet()) {

					log.debug(String.format("using xslt param: %s - %s", e.getKey(), e.getValue()));

					t.setParameter(e.getKey(), e.getValue());

				}

			}

			return new XsltRowTransformer(t);

		} catch (final Exception e) {

			throw new RuntimeException("Error generating transformer from xslt:\n" + xslt, e);

		}

	}

	public static XsltRowTransformer newInstance(final String xslt) {

		return new XsltRowTransformerFactory().getTransformer(xslt);

	}

	public static XsltRowTransformer newInstance(final String xslt, final Map<String, Object> params) {

		return new XsltRowTransformerFactory().getTransformer(xslt, params);

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:p="http://www.springframework.org/schema/p"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">

	<bean id="configurationEnumerator" class="eu.dnetlib.data.hadoop.config.ConfigurationEnumerator" />

	<bean id="DM" class="eu.dnetlib.data.hadoop.config.ConfigurationFactory"

		p:defaults="${services.data.hadoop.dm.properties}" />

	<bean id="IIS" class="eu.dnetlib.data.hadoop.config.ConfigurationFactory"

		p:defaults="${services.data.hadoop.iis.properties}" />

	<bean id="sequenceFileWriterFactory"

		class="eu.dnetlib.data.hadoop.hdfs.SequenceFileWriterFactory"

		scope="prototype" p:keyClass="${services.data.hadoop.hdfs.seqfilewriterfactory.keyclass}"

		p:valueClass="${services.data.hadoop.hdfs.seqfilewriterfactory.valueclass}"

		p:compressionType="${services.data.hadoop.hdfs.seqfilewriterfactory.compressiontype}"

		p:blockSize="${services.data.hadoop.hdfs.seqfilewriterfactory.blocksize}" 

		p:replication="${services.data.hadoop.hdfs.seqfilewriterfactory.replication}"/>

</beans>

dnet.clustername				=	DM

#CORE-SITE

fs.defaultFS					=	hdfs://localhost:8020

hadoop.security.authentication	=	simple

hadoop.security.auth_to_local	=	DEFAULT

hadoop.rpc.socket.factory.class.default	=	org.apache.hadoop.net.StandardSocketFactory

#HBASE-SITE

hbase.rootdir					=	hdfs://localhost:8020/hbase

hbase.security.authentication	=	simple

zookeeper.znode.rootserver		=	root-region-server

hbase.zookeeper.quorum			=	localhost

hbase.zookeeper.property.clientPort	=	2181

hbase.zookeeper.client.port		=	2181

zookeeper.znode.parent			=	/hbase

#HDFS-SITE

dfs.replication					=	2

dfs.namenode.servicerpc-address	=	localhost:8022

dfs.namenode.http-address		=	localhost:50070

#MAPRED-SITE

mapred.job.tracker					=	localhost:8021

#OOZIE SERVER 

oozie.service.loc				=	http://localhost:11000/oozie

######### CDH QUICKSTART

#HDFS-SITE

#dfs.namenode.servicerpc-address=cloudera-vm:8022

#dfs.https.port=50470

#dfs.namenode.http-address=cloudera-vm:50070

#dfs.replication=3

#dfs.blocksize=134217728

#dfs.client.use.datanode.hostname=false

#fs.permissions.umask-mode=022

#dfs.client.read.shortcircuit=false

#dfs.domain.socket.path=/var/run/hdfs-sockets/dn

#dfs.client.read.shortcircuit.skip.checksum=false

#dfs.client.domain.socket.data.traffic=false

#dfs.datanode.hdfs-blocks-metadata.enabled=true

#

#

##HBASE-SITE

#hbase.rootdir=hdfs://cloudera-vm:8020/hbase

#hbase.client.write.buffer=2097152

#hbase.client.pause=1000

#hbase.client.retries.number=10

#hbase.client.scanner.caching=1

#hbase.client.keyvalue.maxsize=10485760

#hbase.rpc.timeout=60000

#hbase.security.authentication=simple

#zookeeper.session.timeout=60000

#zookeeper.znode.parent=/hbase

#zookeeper.znode.rootserver=root-region-server

#hbase.zookeeper.quorum=cloudera-vm

#hbase.zookeeper.property.clientPort=2181

#

##CORE-SITE

#fs.defaultFS=hdfs://cloudera-vm:8020

#fs.trash.interval=1

#hadoop.security.authentication=simple

#hadoop.rpc.protection=authentication

#hadoop.security.auth_to_local=DEFAULT

dnet.clustername				=	IIS

#CORE-SITE

fs.defaultFS					=	hdfs://iis-cluster-nn

hadoop.security.authentication	=	simple

hadoop.security.auth_to_local	=	DEFAULT

#HBASE-SITE

hbase.rootdir					=	hdfs://iis-cluster-nn/hbase

hbase.security.authentication	=	simple

zookeeper.znode.rootserver		=	root-region-server

hbase.zookeeper.quorum			=	master1.hadoop.iis.openaire.eu,master2.hadoop.iis.openaire.eu,master3.hadoop.iis.openaire.eu

hbase.zookeeper.property.clientPort	=	2181

hbase.zookeeper.client.port		=	2181

zookeeper.znode.parent			=	/hbase

#HDFS-SITE

dfs.replication					=	2

dfs.nameservices				=	iis-cluster-nn

dfs.ha.namenodes.iis-cluster-nn	=	nn1,nn2

dfs.namenode.rpc-address.iis-cluster-nn.nn1=namenode1.hadoop.iis.openaire.eu:8020

dfs.namenode.http-address.iis-cluster-nn.nn1=namenode1.hadoop.iis.openaire.eu:50070

dfs.namenode.rpc-address.iis-cluster-nn.nn2=namenode2.hadoop.iis.openaire.eu:8020

dfs.namenode.http-address.iis-cluster-nn.nn2=namenode2.hadoop.iis.openaire.eu:50070

dfs.client.failover.proxy.provider.iis-cluster-nn=org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider

#MAPRED-SITE

mapred.job.tracker					=	iis-cluster-jt

mapred.jobtrackers.iis-cluster-jt	=	jt1,jt2

mapred.jobtracker.rpc-address.iis-cluster-jt.jt1 = master1.hadoop.iis.openaire.eu:8021

mapred.jobtracker.rpc-address.iis-cluster-jt.jt2 = master1.hadoop.iis.openaire.eu:8022

mapred.client.failover.proxy.provider.iis-cluster-jt = org.apache.hadoop.mapred.ConfiguredFailoverProxyProvider

mapred.mapper.new-api			=	true

mapred.reducer.new-api			=	true

#OOZIE SERVER

oozie.service.loc				=	http://oozie.hadoop.iis.openaire.eu:11000/oozie

dnet.clustername				=	IIS

#CORE-SITE

fs.defaultFS				    =	hdfs://nameservice1

#HDFS-SITE

dfs.replication					=	2

#MAPRED-SITE

jobTracker                      =   yarnRM

mapred.job.tracker              =   yarnRM

mapred.mapper.new-api			=	true

mapred.reducer.new-api			=	true

#OOZIE SERVER

oozie.service.loc               =   http://iis-cdh5-test-m3.ocean.icm.edu.pl:11000/oozie

#MISC

importerQueueName               =   dm_import

oozieLauncherQueueName          =   default

dnet.clustername				=	DM

#CORE-SITE

fs.defaultFS					=	hdfs://nmis-hadoop-cluster

hadoop.security.authentication	=	simple

hadoop.security.auth_to_local	=	DEFAULT

hadoop.rpc.socket.factory.class.default	=	org.apache.hadoop.net.StandardSocketFactory

#HBASE-SITE

hbase.rootdir					=	hdfs://nmis-hadoop-cluster/hbase

hbase.security.authentication	=	simple

zookeeper.znode.rootserver		=	root-region-server

hbase.zookeeper.quorum			=	quorum1.t.hadoop.research-infrastructures.eu,quorum2.t.hadoop.research-infrastructures.eu,quorum3.t.hadoop.research-infrastructures.eu,quorum4.t.hadoop.research-infrastructures.eu,jobtracker.t.hadoop.research-infrastructures.eu

hbase.zookeeper.property.clientPort	=	2182

hbase.zookeeper.client.port		=	2182

zookeeper.znode.parent			=	/hbase

#HDFS-SITE

dfs.replication					=	2

dfs.nameservices						=	nmis-hadoop-cluster

dfs.ha.namenodes.nmis-hadoop-cluster	=	nn1,nn2

dfs.namenode.rpc-address.nmis-hadoop-cluster.nn1=quorum1.t.hadoop.research-infrastructures.eu:8020

dfs.namenode.http-address.nmis-hadoop-cluster.nn1=quorum1.t.hadoop.research-infrastructures.eu:50070

dfs.namenode.rpc-address.nmis-hadoop-cluster.nn2=quorum2.t.hadoop.research-infrastructures.eu:8020

dfs.namenode.http-address.nmis-hadoop-cluster.nn2=quorum2.t.hadoop.research-infrastructures.eu:50070

dfs.client.failover.proxy.provider.nmis-hadoop-cluster=org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider

#MAPRED-SITE

mapred.job.tracker					=	nmis-hadoop-jt

mapred.jobtrackers.nmis-hadoop-jt	=	jt1,jt2

mapred.jobtracker.rpc-address.nmis-hadoop-jt.jt1 = jobtracker.t.hadoop.research-infrastructures.eu:8021

mapred.jobtracker.rpc-address.nmis-hadoop-jt.jt2 = quorum4.t.hadoop.research-infrastructures.eu:8022

mapred.client.failover.proxy.provider.nmis-hadoop-jt = org.apache.hadoop.mapred.ConfiguredFailoverProxyProvider

mapred.mapper.new-api			=	true

mapred.reducer.new-api			=	true

#OOZIE SERVER

oozie.service.loc				=	http://oozie.t.hadoop.research-infrastructures.eu:11000/oozie

services.data.hadoop.dm.properties = classpath:/eu/dnetlib/data/hadoop/config/hadoop-default.dm.cnr.properties

services.data.hadoop.iis.properties = classpath:/eu/dnetlib/data/hadoop/config/hadoop-default.iis.icm.properties

services.data.hadoop.hdfs.seqfilewriterfactory.keyclass = org.apache.hadoop.io.Text

services.data.hadoop.hdfs.seqfilewriterfactory.valueclass = org.apache.hadoop.io.Text

services.data.hadoop.hdfs.seqfilewriterfactory.compressiontype = BLOCK

services.data.hadoop.hdfs.seqfilewriterfactory.blocksize = 16M

services.data.hadoop.hdfs.seqfilewriterfactory.replication = 3

dnet.clustername				=	DM

#CORE-SITE

fs.defaultFS					=	hdfs://d-net2.house.cnr.it:8020

hadoop.security.authentication	=	simple

hadoop.security.auth_to_local	=	DEFAULT

hadoop.rpc.socket.factory.class.default	=	org.apache.hadoop.net.StandardSocketFactory

#HBASE-SITE

hbase.rootdir					=	hdfs://d-net2.house.cnr.it:8020/hbase

hbase.security.authentication	=	simple

zookeeper.znode.rootserver		=	root-region-server

hbase.zookeeper.quorum			=	d-net2.house.cnr.it

hbase.zookeeper.property.clientPort	=	2181

hbase.zookeeper.client.port		=	2181

zookeeper.znode.parent			=	/hbase

#HDFS-SITE

dfs.replication					=	2

dfs.namenode.servicerpc-address	=	d-net2.house.cnr.it:8022

dfs.namenode.http-address		=	d-net2.house.cnr.it:50070

dfs.replication					= 	1

#MAPRED-SITE

mapred.job.tracker				=	d-net2.house.cnr.it:8021

mapred.child.java.opts			=	-Djava.net.preferIPv4Stack=true -Xmx1073741824

mapred.mapper.new-api			=	true

mapred.reducer.new-api			=	true

#OOZIE SERVER

#oozie.service.loc				=	http://oozie.t.hadoop.research-infrastructures.eu:11000/oozie

dnet.clustername				=	DM

#CORE-SITE

fs.defaultFS					=	hdfs://dm-cluster-nn

hadoop.security.authentication	=	simple

hadoop.security.auth_to_local	=	DEFAULT

hadoop.rpc.socket.factory.class.default	=	org.apache.hadoop.net.StandardSocketFactory

#HBASE-SITE

hbase.rootdir					=	hdfs://dm-cluster-nn/hbase

hbase.security.authentication	=	simple

zookeeper.znode.rootserver		=	root-region-server

hbase.zookeeper.quorum			=	namenode1.hadoop.dm.openaire.eu,namenode2.hadoop.dm.openaire.eu,jobtracker1.hadoop.dm.openaire.eu,jobtracker2.hadoop.dm.openaire.eu,hbase-master1.hadoop.dm.openaire.eu

hbase.zookeeper.property.clientPort	=	2181

hbase.zookeeper.client.port		=	2181

zookeeper.znode.parent			=	/hbase

#HDFS-SITE

dfs.replication					=	2

dfs.nameservices				=	dm-cluster-nn

dfs.ha.namenodes.dm-cluster-nn	=	nn1,nn2

dfs.namenode.rpc-address.dm-cluster-nn.nn1=namenode1.hadoop.dm.openaire.eu:8020

dfs.namenode.http-address.dm-cluster-nn.nn1=namenode1.hadoop.dm.openaire.eu:50070

dfs.namenode.rpc-address.dm-cluster-nn.nn2=namenode2.hadoop.dm.openaire.eu:8020

dfs.namenode.http-address.dm-cluster-nn.nn2=namenode2.hadoop.dm.openaire.eu:50070

dfs.client.failover.proxy.provider.dm-cluster-nn=org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider

#MAPRED-SITE

mapred.job.tracker					=	dm-cluster-jt

mapred.jobtrackers.dm-cluster-jt	=	jt1,jt2

mapred.jobtracker.rpc-address.dm-cluster-jt.jt1 = jobtracker1.hadoop.dm.openaire.eu:8021

mapred.jobtracker.rpc-address.dm-cluster-jt.jt2 = jobtracker2.hadoop.dm.openaire.eu:8021

mapred.client.failover.proxy.provider.dm-cluster-jt = org.apache.hadoop.mapred.ConfiguredFailoverProxyProvider

mapred.mapper.new-api			=	true

mapred.reducer.new-api			=	true

#OOZIE SERVER 

oozie.service.loc				=	http://oozie.hadoop.dm.openaire.eu:11000/oozie

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-hadoop-commons</artifactId>

	<packaging>jar</packaging>

	<version>2.0.1</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-hadoop-commons/tags/dnet-hadoop-commons-2.0.1</developerConnection>

	</scm>

	<repositories>

		<!-- Cloudera Repositories -->

		<repository>

			<snapshots>

				<enabled>false</enabled>

			</snapshots>

			<id>cloudera-central</id>

			<name>cloundera-libs-release</name>

			<url>http://maven.research-infrastructures.eu/nexus/content/repositories/cloudera-central</url>

		</repository>

		<repository>

			<id>cloudera-snapshots</id>

			<name>cloudera-libs-snapshot</name>

			<url>http://maven.research-infrastructures.eu/nexus/content/repositories/cloudera-snapshots</url>

		</repository>

		<repository>

			<id>typesafe</id>

			<name>typesafe-releases</name>

			<url>http://maven.research-infrastructures.eu/nexus/content/repositories/typesafe</url>

		</repository>

	</repositories>

	<dependencies>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-misc-utils</artifactId>

			<version>[1.0.0,2.0.0)</version>

			<exclusions>

				<exclusion>

					<artifactId>slf4j-api</artifactId>

					<groupId>org.slf4j</groupId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>org.apache.hadoop</groupId>

			<artifactId>hadoop-common</artifactId>

			<version>${hadoop.common.version}</version>

			<exclusions>

                                <exclusion>

                                        <artifactId>mockito-all</artifactId>

                                        <groupId>org.mockito</groupId>

                                </exclusion>

                                <exclusion>

                                        <artifactId>jetty</artifactId>

                                        <groupId>org.mortbay.jetty</groupId>

                                </exclusion>

                                <exclusion>

                                        <artifactId>jetty-util</artifactId>

                                        <groupId>org.mortbay.jetty</groupId>

                                </exclusion>

                                <exclusion>

                                        <artifactId>core</artifactId>

                                        <groupId>org.eclipse.jdt</groupId>

                                </exclusion>

				<exclusion>

					<groupId>javax.servlet</groupId>

					<artifactId>servlet-api</artifactId>

				</exclusion>

				<exclusion>

					<artifactId>slf4j-api</artifactId>

					<groupId>org.slf4j</groupId>

				</exclusion>

				<exclusion>

					<artifactId>slf4j-log4j12</artifactId>

					<groupId>org.slf4j</groupId>

				</exclusion>

				<exclusion>

					<artifactId>jsp-api</artifactId>

					<groupId>javax.servlet.jsp</groupId>

				</exclusion>

				<exclusion>

					<artifactId>guava</artifactId>

					<groupId>com.google.guava</groupId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>org.apache.hadoop</groupId>

			<artifactId>hadoop-core</artifactId>

			<version>${hadoop.core.version}</version>

			<exclusions>

                                <exclusion>

                                        <artifactId>core</artifactId>

                                        <groupId>org.eclipse.jdt</groupId>

                                </exclusion>

                                <exclusion>

                                        <artifactId>jetty</artifactId>

                                        <groupId>org.mortbay.jetty</groupId>

                                </exclusion>

                                <exclusion>

                                        <artifactId>jetty-util</artifactId>

                                        <groupId>org.mortbay.jetty</groupId>

                                </exclusion>

				<exclusion>

					<groupId>tomcat</groupId>

					<artifactId>jasper-runtime</artifactId>

				</exclusion>

				<exclusion>

					<groupId>tomcat</groupId>

					<artifactId>jasper-compiler</artifactId>

				</exclusion>

				<exclusion>

					<artifactId>jsp-api</artifactId>

					<groupId>javax.servlet.jsp</groupId>

				</exclusion>

				<exclusion>

					<groupId>javax.servlet</groupId>

					<artifactId>servlet-api</artifactId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>org.springframework</groupId>

			<artifactId>spring-beans</artifactId>

			<version>${spring.version}</version>

		</dependency>

		<!-- DO NOT REMOVE THIS DEPENDENCY!!! MODULES ACCESSING HDFS NEEDS IT EVEN 

			IF mvn dependency:analyze SUGGESTS TO REMOVE IT -->

		<dependency>

			<groupId>org.apache.hadoop</groupId>

			<artifactId>hadoop-hdfs</artifactId>

			<version>${hadoop.hdfs.version}</version>

			<exclusions>

                                <exclusion>

                                        <artifactId>jasper-runtime</artifactId>

                                        <groupId>tomcat</groupId>

                                </exclusion>

                                <exclusion>

                                        <artifactId>jetty</artifactId>

                                        <groupId>org.mortbay.jetty</groupId>

                                </exclusion>

                                <exclusion>

                                        <artifactId>jetty-util</artifactId>

                                        <groupId>org.mortbay.jetty</groupId>

                                </exclusion>

				<exclusion>

					<groupId>javax.servlet</groupId>

					<artifactId>servlet-api</artifactId>

				</exclusion>

				<exclusion>

					<artifactId>jsp-api</artifactId>

					<groupId>javax.servlet.jsp</groupId>

				</exclusion>

				<exclusion>

					<artifactId>guava</artifactId>

					<groupId>com.google.guava</groupId>

				</exclusion>

			</exclusions>

		</dependency>

	</dependencies>

</project>