Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/Process.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/Process.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/Process.java	(revision 51263)
@@ -0,0 +1,43 @@
+package eu.dnetlib.dhp.common.java;
+
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+
+import eu.dnetlib.dhp.common.java.porttype.PortType;
+
+/** Workflow node written in Java.
+ * 
+ * The implementing class has to define a constructor with no parameters 
+ * (possibly the default one) or a constructor with String[] as a single
+ * parameter.
+ * @author Mateusz Kobos
+ */
+public interface Process {
+	/**
+	 * Run the process.
+	 * 
+	 * The process ends with a success status if no exception is thrown, 
+	 * otherwise it ends with an error status.
+	 *
+	 * @param parameters parameters of the process. Each parameter
+	 * corresponds to a single entry in the map, its name is the key, its
+	 * value is the value.
+	 * @throws Exception if thrown, it means that the process finished
+	 * with an error status
+	 */
+	void run(PortBindings portBindings, Configuration conf, 
+			Map<String, String> parameters) throws Exception;
+	
+	/**
+	 * @return map containing as the key: name of the port, as the value: type
+	 * of the port 
+	 */
+	Map<String, PortType> getInputPorts();
+	
+	/**
+	 * @return map containing as the key: name of the port, as the value: type
+	 * of the port 
+	 */
+	Map<String, PortType> getOutputPorts();
+}
\ No newline at end of file
Index: dnet-hadoop/dhp-common/dhp-common.iml
===================================================================
--- dnet-hadoop/dhp-common/dhp-common.iml	(nonexistent)
+++ dnet-hadoop/dhp-common/dhp-common.iml	(revision 51263)
@@ -0,0 +1,306 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
+    <output url="file://$MODULE_DIR$/target/classes" />
+    <output-test url="file://$MODULE_DIR$/target/test-classes" />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
+      <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
+      <excludeFolder url="file://$MODULE_DIR$/target" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="module" module-name="dhp-schemas" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.oozie:oozie-core:4.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.oozie:oozie-hbase:1.2.0-cdh5.9.0.oozie-4.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hbase:hbase-server:1.2.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hbase:hbase-procedure:1.2.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hbase:hbase-common:tests:1.2.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hbase:hbase-prefix-tree:1.2.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.lmax:disruptor:3.3.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.oozie:oozie-hcatalog:1.1.0-cdh5.9.0.oozie-4.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hive.hcatalog:hive-hcatalog-server-extensions:1.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hive.hcatalog:hive-hcatalog-core:1.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hive.hcatalog:hive-webhcat-java-client:1.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hive:hive-common:1.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.eclipse.jetty.aggregate:jetty-all:7.6.0.v20120127" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.geronimo.specs:geronimo-jaspic_1.0_spec:1.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.geronimo.specs:geronimo-annotation_1.0_spec:1.1.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: asm:asm-commons:3.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: asm:asm-tree:3.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.ant:ant:1.9.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.ant:ant-launcher:1.9.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hive:hive-metastore:1.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.jolbox:bonecp:0.8.0.RELEASE" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.datanucleus:datanucleus-api-jdo:3.2.6" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.datanucleus:datanucleus-core:3.2.10" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.datanucleus:datanucleus-rdbms:3.2.9" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: javax.jdo:jdo-api:3.0.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: javax.transaction:jta:1.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.antlr:antlr-runtime:3.4" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.antlr:stringtemplate:3.2.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: antlr:antlr:2.7.7" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hive:hive-exec:1.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hive:hive-ant:1.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.cloudera.logredactor:logredactor:1.0.3" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.antlr:ST4:4.0.4" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.codehaus.groovy:groovy-all:2.4.4" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: stax:stax-api:1.0.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hive:hive-serde:1.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.sf.opencsv:opencsv:2.3" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.twitter:parquet-hadoop-bundle:1.5.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.sun.jersey:jersey-servlet:1.14" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.thrift:libfb303:0.9.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.oozie:oozie-client:4.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.activemq:activemq-client:5.13.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.geronimo.specs:geronimo-jms_1.1_spec:1.1.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.fusesource.hawtbuf:hawtbuf:1.11" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.geronimo.specs:geronimo-j2ee-management_1.1_spec:1.0.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: xerces:xercesImpl:2.10.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: xml-apis:xml-apis:1.4.01" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.openjpa:openjpa-persistence:2.2.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.openjpa:openjpa-kernel:2.2.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.openjpa:openjpa-lib:2.2.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.sourceforge.serp:serp:1.14.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.geronimo.specs:geronimo-jta_1.1_spec:1.1.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.geronimo.specs:geronimo-jpa_2.0_spec:1.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.openjpa:openjpa-jdbc:2.2.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.openjpa:openjpa-persistence-jdbc:2.2.2" level="project" />
+    <orderEntry type="library" name="Maven: commons-lang:commons-lang:2.4" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.sf.ehcache:ehcache-core:2.6.3" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.googlecode.json-simple:json-simple:1.1.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: log4j:apache-log4j-extras:1.2.17" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: commons-el:commons-el:1.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: commons-dbcp:commons-dbcp:1.4" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: commons-pool:commons-pool:1.5.4" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: hsqldb:hsqldb:1.8.0.10" level="project" />
+    <orderEntry type="library" name="Maven: commons-logging:commons-logging:1.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: commons-httpclient:commons-httpclient:3.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: javax.servlet.jsp:jsp-api:2.0" level="project" />
+    <orderEntry type="library" name="Maven: org.mortbay.jetty:jetty:6.1.26.cloudera.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.derby:derby:10.10.1.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: javax.mail:mail:1.4" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: javax.activation:activation:1.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: postgresql:postgresql:9.0-801.jdbc4" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.oozie:oozie-sharelib-oozie:4.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.oozie:oozie-hadoop-utils:2.6.0-cdh5.9.0.oozie-4.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.oozie:oozie-sharelib-hcatalog:4.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hive.hcatalog:hive-hcatalog-pig-adapter:1.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.oozie:oozie-sharelib-distcp:4.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.oozie:oozie-hadoop-distcp:2.6.0-cdh5.9.0.oozie-4.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-distcp:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.quartz-scheduler:quartz:2.1.7" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: c3p0:c3p0:0.9.1.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.curator:curator-recipes:2.7.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.curator:curator-framework:2.7.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.curator:curator-x-discovery:2.7.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.guava:guava:12.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.codahale.metrics:metrics-core:3.0.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.codahale.metrics:metrics-jvm:3.0.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.codahale.metrics:metrics-json:3.0.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.sf.jung:jung-graph-impl:2.0.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.sf.jung:jung-api:2.0.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.sf.jung:jung-visualization:2.0.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.sf.jung:jung-algorithms:2.0.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.cloudera:collections-generic:4.01-cloudera" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.eclipse.jgit:org.eclipse.jgit:3.3.1.201403241930-r" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.googlecode.javaewah:JavaEWAH:0.7.9" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hive:hive-jdbc:1.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hive:hive-service:1.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.sf.jpam:jpam:1.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.jamon:jamon-runtime:2.3.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hive:hive-shims:1.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hive.shims:hive-shims-common:1.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hive.shims:hive-shims-0.23:1.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-yarn-server-resourcemanager:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-yarn-server-web-proxy:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hive.shims:hive-shims-scheduler:1.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.httpcomponents:httpclient:4.2.5" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.httpcomponents:httpcore:4.2.5" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.thrift:libthrift:0.9.3" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-core:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-yarn-common:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-yarn-api:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: javax.xml.bind:jaxb-api:2.2.2" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: javax.xml.stream:stax-api:1.0-2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.sun.jersey:jersey-client:1.9" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.codehaus.jackson:jackson-jaxrs:1.8.8" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.codehaus.jackson:jackson-xc:1.8.8" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.inject:guice:3.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: javax.inject:javax.inject:1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: aopalliance:aopalliance:1.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.sun.jersey.contribs:jersey-guice:1.9" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.protobuf:protobuf-java:2.5.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-annotations:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.inject.extensions:guice-servlet:3.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: io.netty:netty:3.10.5.Final" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-common:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: commons-cli:commons-cli:1.3.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.commons:commons-math3:3.1.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: xmlenc:xmlenc:0.52" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: commons-codec:commons-codec:1.4" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: commons-net:commons-net:3.1" level="project" />
+    <orderEntry type="library" name="Maven: commons-collections:commons-collections:3.2.2" level="project" />
+    <orderEntry type="library" name="Maven: org.mortbay.jetty:jetty-util:6.1.26.cloudera.4" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.sun.jersey:jersey-core:1.9" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.sun.jersey:jersey-json:1.9" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.codehaus.jettison:jettison:1.1" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: com.sun.xml.bind:jaxb-impl:2.2.7" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: com.sun.xml.bind:jaxb-core:2.2.7" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: com.sun.istack:istack-commons-runtime:2.16" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: com.sun.xml.fastinfoset:FastInfoset:1.2.12" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: javax.xml.bind:jsr173_api:1.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.sun.jersey:jersey-server:1.9" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: asm:asm:3.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: tomcat:jasper-compiler:5.5.23" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: tomcat:jasper-runtime:5.5.23" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.java.dev.jets3t:jets3t:0.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.jamesmurty.utils:java-xmlbuilder:0.4" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: commons-configuration:commons-configuration:1.6" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: commons-digester:commons-digester:1.8" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: commons-beanutils:commons-beanutils-core:1.8.0" level="project" />
+    <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-core-asl:1.8.8" level="project" />
+    <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-mapper-asl:1.8.8" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.code.gson:gson:2.8.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-auth:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.directory.server:apacheds-i18n:2.0.0-M15" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.directory.api:api-asn1-api:1.0.0-M20" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.directory.api:api-util:1.0.0-M20" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.jcraft:jsch:0.1.42" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.curator:curator-client:2.7.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.htrace:htrace-core4:4.0.1-incubating" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.zookeeper:zookeeper:3.4.5-cdh5.9.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.4.1" level="project" />
+    <orderEntry type="library" name="Maven: org.tukaani:xz:1.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.spark:spark-core_2.10:1.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.twitter:chill_2.10:0.5.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.esotericsoftware.kryo:kryo:2.21" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.esotericsoftware.reflectasm:reflectasm:shaded:1.07" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.esotericsoftware.minlog:minlog:1.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.twitter:chill-java:0.5.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.xbean:xbean-asm5-shaded:4.4" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-client:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-hdfs:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-app:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-common:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-yarn-client:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-yarn-server-common:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-shuffle:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-jobclient:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-aws:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.amazonaws:aws-java-sdk-s3:1.10.6" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.amazonaws:aws-java-sdk-kms:1.10.6" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.amazonaws:aws-java-sdk-core:1.10.6" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.amazonaws:aws-java-sdk-sts:1.10.6" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.spark:spark-launcher_2.10:1.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.spark:spark-network-common_2.10:1.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.spark:spark-network-shuffle_2.10:1.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.fusesource.leveldbjni:leveldbjni-all:1.8" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.2.3" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.spark:spark-unsafe_2.10:1.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.eclipse.jetty.orbit:javax.servlet:3.0.0.v201112011016" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.slf4j:jul-to-slf4j:1.7.5" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.slf4j:jcl-over-slf4j:1.7.5" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.ning:compress-lzf:1.0.3" level="project" />
+    <orderEntry type="library" name="Maven: org.xerial.snappy:snappy-java:1.0.4.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.jpountz.lz4:lz4:1.3.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.roaringbitmap:RoaringBitmap:0.5.11" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.spark-project.akka:akka-remote_2.10:2.2.3-shaded-protobuf" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.spark-project.akka:akka-actor_2.10:2.2.3-shaded-protobuf" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.typesafe:config:1.0.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.spark-project.protobuf:protobuf-java:2.4.1-shaded" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.uncommons.maths:uncommons-maths:1.2.2a" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.spark-project.akka:akka-slf4j_2.10:2.2.3-shaded-protobuf" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.scala-lang:scala-library:2.10.6" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.json4s:json4s-jackson_2.10:3.2.10" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.json4s:json4s-core_2.10:3.2.10" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.json4s:json4s-ast_2.10:3.2.10" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.scala-lang:scalap:2.10.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.scala-lang:scala-compiler:2.10.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.mesos:mesos:shaded-protobuf:0.21.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: io.netty:netty-all:4.0.29.Final" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.clearspring.analytics:stream:2.7.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: io.dropwizard.metrics:metrics-core:3.1.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: io.dropwizard.metrics:metrics-jvm:3.1.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: io.dropwizard.metrics:metrics-json:3.1.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: io.dropwizard.metrics:metrics-graphite:3.1.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.2.3" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.fasterxml.jackson.core:jackson-core:2.2.3" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.fasterxml.jackson.module:jackson-module-scala_2.10:2.2.3" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.ivy:ivy:2.4.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: oro:oro:2.0.8" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.tachyonproject:tachyon-client:0.8.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.tachyonproject:tachyon-underfs-hdfs:0.8.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.tachyonproject:tachyon-underfs-s3:0.8.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.tachyonproject:tachyon-underfs-local:0.8.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.razorvine:pyrolite:4.9" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.sf.py4j:py4j:0.9" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.intel.chimera:chimera:0.9.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.spark-project.spark:unused:1.0.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.spark:spark-sql_2.10:1.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.spark:spark-catalyst_2.10:1.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.scala-lang:scala-reflect:2.10.5" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.codehaus.janino:janino:2.7.8" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.codehaus.janino:commons-compiler:2.7.8" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.twitter:parquet-column:1.5.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.twitter:parquet-common:1.5.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.twitter:parquet-encoding:1.5.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.twitter:parquet-hadoop:1.5.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.twitter:parquet-format:2.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.twitter:parquet-jackson:1.5.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.avro:avro:1.7.6-cdh5.9.0" level="project" />
+    <orderEntry type="library" name="Maven: com.thoughtworks.paranamer:paranamer:2.8" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.avro:avro-mapred:hadoop2:1.7.6-cdh5.9.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.avro:avro-ipc:1.7.6-cdh5.9.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.velocity:velocity:1.7" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.avro:avro-ipc:tests:1.7.6-cdh5.9.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.5" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework:spring-beans:4.2.5.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework:spring-core:4.2.5.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: com.beust:jcommander:1.60" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.pig:pig:0.12.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hbase:hbase-client:1.2.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hbase:hbase-annotations:1.2.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hbase:hbase-common:1.2.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-core:2.6.0-mr1-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hbase:hbase-protocol:1.2.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.htrace:htrace-core:3.2.0-incubating" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.jruby.jcodings:jcodings:1.0.8" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.jruby.joni:joni:2.1.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.yammer.metrics:metrics-core:2.2.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hbase:hbase-client:tests:1.2.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.mortbay.jetty:jsp-api-2.1:6.1.14" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.mortbay.jetty:jsp-2.1:6.1.14" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.eclipse.jdt:core:3.1.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: ant:ant:1.6.5" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.python:jython-standalone:2.5.2" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.sf.kosmosfs:kfs:0.3" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: jline:jline:2.11" level="project" />
+    <orderEntry type="library" name="Maven: joda-time:joda-time:1.6" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: dk.brics.automaton:automaton:1.11-8" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.fusesource.jansi:jansi:1.9" level="project" />
+    <orderEntry type="library" name="Maven: com.linkedin.datafu:datafu:1.2.0" level="project" />
+    <orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils:1.9.3" level="project" />
+    <orderEntry type="library" name="Maven: commons-io:commons-io:2.5" level="project" />
+    <orderEntry type="library" name="Maven: org.jdom:jdom:1.1.3" level="project" />
+    <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.22" level="project" />
+    <orderEntry type="library" name="Maven: org.slf4j:slf4j-log4j12:1.7.22" level="project" />
+    <orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: javax.servlet:javax.servlet-api:3.1.0" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.12" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-all:1.3" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.mockito:mockito-all:1.10.19" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.powermock:powermock-core:1.6.6" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.powermock:powermock-reflect:1.6.6" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: org.objenesis:objenesis:2.4" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.javassist:javassist:3.21.0-GA" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.code.findbugs:annotations:3.0.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.jcip:jcip-annotations:1.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.code.findbugs:jsr305:3.0.1" level="project" />
+  </component>
+</module>
\ No newline at end of file
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/counter/NamedCountersFileWriter.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/counter/NamedCountersFileWriter.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/counter/NamedCountersFileWriter.java	(revision 51263)
@@ -0,0 +1,52 @@
+package eu.dnetlib.dhp.common.counter;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Properties;
+
+/**
+ * Writer of {@link NamedCounters} object into a properties file.
+ * 
+ * @author madryk
+ */
+public class NamedCountersFileWriter {
+    
+    
+    //------------------------ LOGIC --------------------------
+    
+    /**
+     * Writes {@link NamedCounters} as a properties file located under
+     * provided filePath.
+     * 
+     * @throws IOException if writing to properties file resulted in an error
+     */
+    public void writeCounters(NamedCounters counters, String filePath) throws IOException {
+        
+        Properties counterProperties = buildPropertiesFromCounters(counters);
+        
+        File file = new File(filePath);
+        try (OutputStream os = new FileOutputStream(file)) {
+            
+            counterProperties.store(os, null);
+            
+        }
+        
+    }
+    
+    
+    //------------------------ PRIVATE --------------------------
+    
+    private Properties buildPropertiesFromCounters(NamedCounters counters) {
+        
+        Properties properties = new Properties();
+        
+        for (String counterName : counters.counterNames()) {
+            long count = counters.currentValue(counterName);
+            properties.put(counterName, String.valueOf(count));
+        }
+        
+        return properties;
+    }
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/counter/NamedCountersAccumulableParam.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/counter/NamedCountersAccumulableParam.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/counter/NamedCountersAccumulableParam.java	(revision 51263)
@@ -0,0 +1,48 @@
+package eu.dnetlib.dhp.common.counter;
+
+import org.apache.spark.AccumulableParam;
+
+import scala.Tuple2;
+
+/**
+ * Spark {@link AccumulableParam} for tracking multiple counter values using {@link NamedCounters}.
+ * 
+ * @author madryk
+ */
+public class NamedCountersAccumulableParam implements AccumulableParam<NamedCounters, Tuple2<String,Long>> {
+
+    private static final long serialVersionUID = 1L;
+
+    
+    //------------------------ LOGIC --------------------------
+    
+    /**
+     * Increments {@link NamedCounters} counter with the name same as the first element of passed incrementValue tuple
+     * by value defined in the second element of incrementValue tuple.
+     */
+    @Override
+    public NamedCounters addAccumulator(NamedCounters counters, Tuple2<String, Long> incrementValue) {
+        counters.increment(incrementValue._1, incrementValue._2);
+        return counters;
+    }
+
+    /**
+     * Merges two passed {@link NamedCounters}.
+     */
+    @Override
+    public NamedCounters addInPlace(NamedCounters counters1, NamedCounters counters2) {
+        for (String counterName2 : counters2.counterNames()) {
+            counters1.increment(counterName2, counters2.currentValue(counterName2));
+        }
+        return counters1;
+    }
+
+    /**
+     * Returns passed initialCounters value without any modifications.
+     */
+    @Override
+    public NamedCounters zero(NamedCounters initialCounters) {
+        return initialCounters;
+    }
+
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/oozie/property/ConditionalPropertySetter.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/oozie/property/ConditionalPropertySetter.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/oozie/property/ConditionalPropertySetter.java	(revision 51263)
@@ -0,0 +1,76 @@
+package eu.dnetlib.dhp.common.oozie.property;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStream;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Properties;
+
+import eu.dnetlib.dhp.common.java.PortBindings;
+import eu.dnetlib.dhp.common.java.Process;
+import eu.dnetlib.dhp.common.java.porttype.PortType;
+import org.apache.hadoop.conf.Configuration;
+
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.OOZIE_ACTION_OUTPUT_FILENAME;
+
+/**
+ * This process is a solution for setting dynamic properties in oozie workflow definition.
+ * 
+ * Expects three parameters to be provided: the first 'condition' parameter is boolean value 
+ * based on which either first 'inCaseOfTrue' or second 'elseCase' parameter value is set as 
+ * the 'result' property.
+ *  
+ * This can be understood as the: 
+ * 
+ * condition ? inCaseOfTrue : elseCase
+ * 
+ * java syntax equivalent.
+ * 
+ * @author mhorst
+ *
+ */
+public class ConditionalPropertySetter implements Process {
+
+	public static final String PARAM_CONDITION = "condition";
+	public static final String PARAM_INCASEOFTRUE = "inCaseOfTrue";
+	public static final String PARAM_ELSECASE = "elseCase";
+	
+	public static final String OUTPUT_PROPERTY_RESULT = "result";
+	
+	@Override
+	public Map<String, PortType> getInputPorts() {
+		return Collections.emptyMap();
+	}
+
+	@Override
+	public Map<String, PortType> getOutputPorts() {
+		return Collections.emptyMap();
+	}
+
+	@Override
+	public void run(PortBindings portBindings, Configuration conf,
+			Map<String, String> parameters) throws Exception {
+
+		String condition = parameters.get(PARAM_CONDITION);
+		if (condition == null) {
+			throw new RuntimeException("unable to make decision: " + 
+					PARAM_CONDITION + " parameter was not set!");
+		}
+
+		Properties props = new Properties();
+        props.setProperty(OUTPUT_PROPERTY_RESULT, 
+        		Boolean.parseBoolean(condition)?
+        				parameters.get(PARAM_INCASEOFTRUE):
+        					parameters.get(PARAM_ELSECASE));
+        OutputStream os = new FileOutputStream(
+        		new File(System.getProperty(OOZIE_ACTION_OUTPUT_FILENAME)));
+        try {
+        	props.store(os, "");	
+        } finally {
+        	os.close();	
+        }
+
+	}
+
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/CharSequenceUtils.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/CharSequenceUtils.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/CharSequenceUtils.java	(revision 51263)
@@ -0,0 +1,31 @@
+package eu.dnetlib.dhp.common.string;
+
+/**
+ * Operations on {@link CharSequence} 
+ * 
+ * @author Łukasz Dumiszewski
+*/
+
+public final class CharSequenceUtils {
+
+    
+    //------------------------ CONSTRUCTORS --------------------------
+    
+    private CharSequenceUtils() {
+        throw new IllegalStateException("may not be initialized");
+    }
+    
+    
+    //------------------------ LOGIC --------------------------
+    
+    /**
+     * Converts the given {@link CharSequence} <code>value</code> to {@link String} by using {@link CharSequence#toString()}.
+     * Returns empty string if <code>value</code> is null.
+     */
+    public static String toStringWithNullToEmpty(CharSequence value) {
+        
+        return value == null? "": value.toString();
+        
+    }
+    
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/counter/NamedCounters.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/counter/NamedCounters.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/counter/NamedCounters.java	(revision 51263)
@@ -0,0 +1,111 @@
+package eu.dnetlib.dhp.common.counter;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Map;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+
+/**
+ * Class that groups several counters which are identified by name (<code>String</code> value).
+ * 
+ * @author madryk
+ */
+public class NamedCounters implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+    
+    
+    private final Map<String, Long> counters;
+    
+    
+    //------------------------ CONSTRUCTORS --------------------------
+    
+    /**
+     * Creates {@link NamedCounters} with empty initial counters.
+     */
+    public NamedCounters() {
+        this.counters = Maps.newHashMap();
+    }
+    
+    /**
+     * Creates {@link NamedCounters} with initial counters.<br/>
+     * Starting value of initial counters is zero.
+     * 
+     * @param initialCounterNames - names of initial counters
+     */
+    public NamedCounters(String[] initialCounterNames) {
+        Preconditions.checkNotNull(initialCounterNames);
+        
+        this.counters = Maps.newHashMap();
+        
+        for (String initialCounterName : initialCounterNames) {
+            this.counters.put(initialCounterName, 0L);
+        }
+    }
+    
+    /**
+     * Creates {@link NamedCounters} with initial counters.<br/>
+     * Starting value of initial counters is zero.
+     * 
+     * @param initialCounterNamesEnumClass - enum class providing names of initial counters
+     */
+    public <E extends Enum<E>> NamedCounters(Class<E> initialCounterNamesEnumClass) {
+        Preconditions.checkNotNull(initialCounterNamesEnumClass);
+        
+        this.counters = Maps.newHashMap();
+        Enum<?>[] enumConstants = initialCounterNamesEnumClass.getEnumConstants();
+        
+        for (int i=0; i<enumConstants.length; ++i) {
+            this.counters.put(enumConstants[i].name(), 0L);
+        }
+        
+    }
+    
+    
+    //------------------------ LOGIC --------------------------
+    
+    /**
+     * Increments value by 1 of a counter with the name specified as parameter.<br/>
+     * Internally uses {@link #increment(String, Long)} 
+     */
+    public void increment(String counterName) {
+        increment(counterName, 1L);
+    }
+    
+    /**
+     * Increments value of a counter with the name specified as parameter by the given value.<br/>
+     * If current instance of {@link NamedCounters} does not contain counter
+     * with provided name, then before incrementing counter will be created with starting
+     * value equal to zero.
+     */
+    public void increment(String counterName, Long incrementValue) {
+        
+        long oldValue = counters.getOrDefault(counterName, 0L);
+        counters.put(counterName, oldValue + incrementValue);
+    }
+    
+    /**
+     * Returns current value of a counter with the name specified as parameter.
+     * 
+     * @throws IllegalArgumentException when {@link NamedCounters} does not contain counter
+     *      with provided name
+     */
+    public long currentValue(String counterName) {
+        
+        if (!counters.containsKey(counterName)) {
+            throw new IllegalArgumentException("Couldn't find counter with name: " + counterName);
+        }
+        
+        return counters.get(counterName);
+    }
+    
+    /**
+     * Returns names of currently tracked counters.
+     */
+    public Collection<String> counterNames() {
+        return counters.keySet();
+    }
+    
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/protobuf/AvroToProtoBufConverter.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/protobuf/AvroToProtoBufConverter.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/protobuf/AvroToProtoBufConverter.java	(revision 51263)
@@ -0,0 +1,12 @@
+package eu.dnetlib.dhp.common.protobuf;
+
+import com.google.protobuf.Message;
+import org.apache.avro.generic.IndexedRecord;
+
+/**
+ * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)
+ */
+public interface AvroToProtoBufConverter<IN extends IndexedRecord, OUT extends Message> {
+    String convertIntoKey(IN datum);
+    OUT convertIntoValue(IN datum);
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/Ports.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/Ports.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/Ports.java	(revision 51263)
@@ -0,0 +1,27 @@
+package eu.dnetlib.dhp.common.java;
+
+import java.util.Map;
+
+import eu.dnetlib.dhp.common.java.porttype.PortType;
+
+/**
+ * A class that groups information about input and output ports, i.e.
+ * their (name of the port -> type of the port) mappings. 
+ * @author Mateusz Kobos
+ */
+public class Ports {
+	private final Map<String, PortType> input;
+	private final Map<String, PortType> output;
+	
+	public Ports(Map<String, PortType> input, Map<String, PortType> output){
+		this.input = input;
+		this.output = output;
+	}
+
+	public Map<String, PortType> getInput() {
+		return input;
+	}
+	public Map<String, PortType> getOutput() {
+		return output;
+	}
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/CmdLineParserForProcessRunParameters.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/CmdLineParserForProcessRunParameters.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/CmdLineParserForProcessRunParameters.java	(revision 51263)
@@ -0,0 +1,100 @@
+package eu.dnetlib.dhp.common.java;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Handles parsing parameters passed to the {@link Process}
+ * @author Mateusz Kobos
+ *
+ */
+public class CmdLineParserForProcessRunParameters {
+	/** Parse the command line arguments.
+	 * 
+	 * @param cmdLine command line arguments
+	 * @param ports names of ports that ought to be extracted from command line
+	 */
+	public ProcessParameters run(CommandLine cmdLine, Ports ports) {
+
+		Properties inputProperties = cmdLine.getOptionProperties(
+				CmdLineParser.inputPrefix);
+		assumePortNamesMatch(CmdLineParser.inputPrefix, inputProperties, 
+				ports.getInput().keySet());
+		Map<String, Path> inputBindings = getBindings(
+				inputProperties, ports.getInput().keySet());
+
+		Properties outputProperties = cmdLine.getOptionProperties(
+				CmdLineParser.outputPrefix);
+		assumePortNamesMatch(CmdLineParser.outputPrefix, outputProperties, 
+				ports.getOutput().keySet());
+		Map<String, Path> outputBindings = getBindings(
+				outputProperties, ports.getOutput().keySet());
+
+		PortBindings bindings = new PortBindings(inputBindings, outputBindings);
+
+		Properties specialProperties = cmdLine.getOptionProperties(
+				CmdLineParser.specialParametersPrefix);
+		assumeContainAllMandatoryParameters(
+				specialProperties, CmdLineParser.mandatorySpecialParameters);
+
+		Properties rawProperties = cmdLine.getOptionProperties(
+				CmdLineParser.processParametersPrefix);
+		Map<String, String> processParameters = new HashMap<String, String>();
+		for(Entry<Object, Object> entry: rawProperties.entrySet()){
+			processParameters.put(
+					(String)entry.getKey(),	(String)entry.getValue());
+		}
+		
+		return new ProcessParameters(bindings, processParameters);
+	}
+	
+	private static void assumeContainAllMandatoryParameters(
+			Properties properties, String[] mandatoryParameters){
+		for(String otherParameter: mandatoryParameters){
+			if(!properties.containsKey(otherParameter)){
+				throw new CmdLineParserException(String.format(
+						"Not all mandatory properties are set using the \"%s\" "
+						+ "option are given, e.g. \"-%s\" parameter is missing",
+						CmdLineParser.specialParametersPrefix, otherParameter));
+			}
+		}
+	}
+	
+	private static void assumePortNamesMatch(String cmdLineParamPrefix,
+			Properties cmdLineProperties, Set<String> portNames) {
+		for (String name : portNames) {
+			if (!cmdLineProperties.containsKey(name)) {
+				throw new CmdLineParserException(String.format(
+					"The port with name \"%s\" is not specified in "
+					+ "command line (command line option \"-%s\" is missing)",
+					name, cmdLineParamPrefix + name));
+			}
+		}
+		for (Object cmdLineKeyObject : cmdLineProperties.keySet()) {
+			String name = (String) cmdLineKeyObject;
+			if (!portNames.contains(name)) {
+				throw new CmdLineParserException(String.format(
+						"A port name \"%s\" which is not specified is given "
+						+ "in the command line "
+						+ "(command line option \"%s\" is excess)",
+						name, cmdLineParamPrefix + name));
+			}
+		}
+	}
+
+	private static Map<String, Path> getBindings(
+			Properties cmdLineProperties, Set<String> portNames) {
+		Map<String, Path> bindings = new HashMap<String, Path>();
+		for (String name : portNames) {
+			Path path = new Path((String) cmdLineProperties.get(name));
+			bindings.put(name, path);
+		}
+		return bindings;
+	}
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/lock/LockManagingProcess.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/lock/LockManagingProcess.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/lock/LockManagingProcess.java	(revision 51263)
@@ -0,0 +1,149 @@
+package eu.dnetlib.dhp.common.lock;
+
+import java.security.InvalidParameterException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.Semaphore;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ha.ZKFailoverController;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.Watcher.Event;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.ZooKeeper;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+
+import eu.dnetlib.dhp.common.java.PortBindings;
+import eu.dnetlib.dhp.common.java.porttype.PortType;
+
+/**
+ * Zookeeper lock managing process. Blocks until lock is released.
+ * 
+ * @author mhorst
+ *
+ */
+public class LockManagingProcess implements eu.dnetlib.dhp.common.java.Process {
+
+	public static final String DEFAULT_ROOT_NODE = "/cache";
+	
+	public static final String NODE_SEPARATOR = "/";
+	
+	public static final String PARAM_ZK_SESSION_TIMEOUT = "zk_session_timeout";
+	
+	public static final String PARAM_NODE_ID = "node_id";
+	
+	public static final String PARAM_LOCK_MODE = "mode";
+	
+	public static enum LockMode {
+		obtain,
+		release
+	}
+	
+	public static final int DEFAULT_SESSION_TIMEOUT = 60000;
+	
+	public static final Logger log = Logger.getLogger(LockManagingProcess.class);
+	
+	@Override
+	public Map<String, PortType> getInputPorts() {
+		return Collections.emptyMap();
+	}
+
+	@Override
+	public Map<String, PortType> getOutputPorts() {
+		return Collections.emptyMap();
+	}
+
+	@Override
+	public void run(PortBindings portBindings, Configuration conf,
+			Map<String, String> parameters) throws Exception {
+	    
+		Preconditions.checkArgument(parameters.containsKey(PARAM_NODE_ID), "node id not provided!");
+		Preconditions.checkArgument(parameters.containsKey(PARAM_LOCK_MODE), "lock mode not provided!");
+
+		String zkConnectionString = conf.get(ZKFailoverController.ZK_QUORUM_KEY);
+		Preconditions.checkArgument(StringUtils.isNotBlank(zkConnectionString), 
+		        "zookeeper quorum is unknown, invalid '%s' property value: %s", ZKFailoverController.ZK_QUORUM_KEY, zkConnectionString);
+
+		int sessionTimeout = parameters.containsKey(PARAM_ZK_SESSION_TIMEOUT)?
+		        Integer.valueOf(parameters.get(PARAM_ZK_SESSION_TIMEOUT)) : DEFAULT_SESSION_TIMEOUT;
+
+		final ZooKeeper zooKeeper = new ZooKeeper(zkConnectionString, sessionTimeout, (e) -> {
+		 // we are not interested in generic events
+		});
+		
+//		initializing root node if does not exist
+		if (zooKeeper.exists(DEFAULT_ROOT_NODE, false) == null) {
+			log.info("initializing root node: " + DEFAULT_ROOT_NODE);
+			zooKeeper.create(DEFAULT_ROOT_NODE, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+			log.info("root node initialized");
+		}
+
+		final String nodePath = generatePath(parameters.get(PARAM_NODE_ID), DEFAULT_ROOT_NODE);
+		
+		final Semaphore semaphore = new Semaphore(1);
+		semaphore.acquire();
+		
+		switch(LockMode.valueOf(parameters.get(PARAM_LOCK_MODE))) {
+		    case obtain: {
+		        obtain(zooKeeper, nodePath, semaphore);
+		        break;
+		    }
+		    case release: {
+		        release(zooKeeper, nodePath);
+		        break;
+		    }
+		    default: {
+		        throw new InvalidParameterException("unsupported lock mode: " + parameters.get(PARAM_LOCK_MODE));
+		    }
+		}
+	}
+	
+	// ------------------------- PRIVATE --------------------------
+	
+	private void obtain(final ZooKeeper zooKeeper, final String nodePath, final Semaphore semaphore) throws KeeperException, InterruptedException {
+        log.info("trying to obtain lock: " + nodePath);
+        if (zooKeeper.exists(nodePath, (event) -> {
+            if (Event.EventType.NodeDeleted == event.getType()) {
+                try {
+                    log.info(nodePath + " lock release detected");
+                    log.info("creating new lock instance: " + nodePath + "...");
+                    zooKeeper.create(nodePath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+                    log.info("lock" + nodePath + " created");
+                    semaphore.release();
+                } catch (KeeperException e) {
+                    throw new RuntimeException(e);
+                } catch (InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        }) == null) {
+            log.info("lock not found, creating new lock instance: " + nodePath);
+            zooKeeper.create(nodePath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+            log.info("lock" + nodePath + " created");
+            semaphore.release();
+        } else {
+            // waiting until node is removed by other lock manager
+            log.info("waiting until lock is released");
+            Stopwatch timer = new Stopwatch().start();
+            semaphore.acquire();
+            log.info("lock released, waited for " + timer.elapsedMillis() + " ms");
+            semaphore.release();
+        }
+	}
+	
+	private void release(final ZooKeeper zooKeeper, final String nodePath) throws InterruptedException, KeeperException {
+	    log.info("removing lock" + nodePath + "...");
+        zooKeeper.delete(nodePath, -1);
+        log.info("lock" + nodePath + " removed");
+	}
+	
+	private static final String generatePath(String nodeId, String rootNode) {
+		return rootNode + NODE_SEPARATOR + nodeId.replace('/', '_');
+	}
+	
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/CmdLineParserForProcessConstruction.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/CmdLineParserForProcessConstruction.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/CmdLineParserForProcessConstruction.java	(revision 51263)
@@ -0,0 +1,45 @@
+package eu.dnetlib.dhp.common.java;
+
+import java.lang.reflect.Constructor;
+
+import org.apache.commons.cli.CommandLine;
+
+/**
+ * Handles parsing the command line arguments provided by the Oozie
+ * to create a {@link Process}
+ * @author Mateusz Kobos
+ *
+ */
+public class CmdLineParserForProcessConstruction {
+	public Process run(CommandLine cmdLine){
+		String[] args = cmdLine.getArgs();
+		if(args.length != 1){
+			throw new CmdLineParserException("The name of the class has "+
+					"to be specified as the first agrument");
+		}
+		String className = args[0];
+		
+		String[] constructorParams = cmdLine.getOptionValues(
+				CmdLineParser.constructorPrefix);
+		if(constructorParams == null){
+			constructorParams = new String[0];
+		}
+		try {
+			Class<?> processClass = Class.forName(className);
+			Constructor<?> processConstructor = null;
+			if(constructorParams.length == 0){
+				try{
+					processConstructor = processClass.getConstructor();
+					return (Process) processConstructor.newInstance();
+				} catch(NoSuchMethodException ex){
+				}
+			}
+			processConstructor = processClass.getConstructor(String[].class);
+			return (Process) processConstructor.newInstance(
+						(Object)constructorParams);
+		} catch (Exception e) {
+			throw new CmdLineParserException(String.format(
+					"Problem while creating class \"%s\"", className), e);
+		}
+	}
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/FsShellPermissions.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/FsShellPermissions.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/FsShellPermissions.java	(revision 51263)
@@ -0,0 +1,106 @@
+package eu.dnetlib.dhp.common;
+
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.LinkedList;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
+import org.springframework.beans.BeanUtils;
+import org.springframework.util.ClassUtils;
+import org.springframework.util.ReflectionUtils;
+
+/**
+ * Extracted from: 
+ * https://github.com/spring-projects/spring-hadoop/blob/master/spring-hadoop-core/src/main/java/org/springframework/data/hadoop/fs/FsShellPermissions.java
+ * 
+ * Utility class for accessing Hadoop FsShellPermissions (which is not public) 
+ * without having to duplicate its code. 
+ * @author Costin Leau
+ *
+ */
+public class FsShellPermissions {
+
+	private static boolean IS_HADOOP_20X = ClassUtils.isPresent("org.apache.hadoop.fs.FsShellPermissions$Chmod",
+			FsShellPermissions.class.getClassLoader());
+
+	public enum Op {
+		CHOWN("-chown"), CHMOD("-chmod"), CHGRP("-chgrp");
+
+		private final String cmd;
+
+		Op(String cmd) {
+			this.cmd = cmd;
+		}
+
+		public String getCmd() {
+			return cmd;
+		}
+	}
+
+	// TODO: move this into Spring Core (but add JDK 1.5 compatibility first)
+	@SafeVarargs
+    static <T> T[] concatAll(T[] first, T[]... rest) {
+		// can add some sanity checks
+		int totalLength = first.length;
+		for (T[] array : rest) {
+			totalLength += array.length;
+		}
+		T[] result = Arrays.copyOf(first, totalLength);
+		int offset = first.length;
+		for (T[] array : rest) {
+			System.arraycopy(array, 0, result, offset, array.length);
+			offset += array.length;
+		}
+		return result;
+	}
+
+	public static void changePermissions(FileSystem fs, Configuration config, 
+			Op op, boolean recursive, String group, String uri) {
+		changePermissions(fs, config, op, recursive, group, new String[] {uri});
+	}
+	
+	public static void changePermissions(FileSystem fs, Configuration config, 
+			Op op, boolean recursive, String group, String... uris) {
+		String[] argvs;
+		if (recursive) {
+			argvs = new String[1];
+			argvs[0] = "-R";
+		} else {
+			argvs = new String[0];
+		}
+		argvs = concatAll(argvs, new String[] { group }, uris);
+
+		// Hadoop 1.0.x
+		if (!IS_HADOOP_20X) {
+			Class<?> cls = ClassUtils.resolveClassName("org.apache.hadoop.fs.FsShellPermissions", config.getClass().getClassLoader());
+			Object[] args = new Object[] { fs, op.getCmd(), argvs, 0, new FsShell(config) };
+
+			Method m = ReflectionUtils.findMethod(cls, "changePermissions", FileSystem.class, String.class, String[].class, int.class, FsShell.class);
+			ReflectionUtils.makeAccessible(m);
+			ReflectionUtils.invokeMethod(m, null, args);
+		}
+		// Hadoop 2.x
+		else {
+			Class<?> cmd = ClassUtils.resolveClassName("org.apache.hadoop.fs.shell.Command", config.getClass().getClassLoader());
+			Class<?> targetClz = ClassUtils.resolveClassName("org.apache.hadoop.fs.FsShellPermissions$Chmod", config.getClass().getClassLoader());
+			Configurable target = (Configurable) BeanUtils.instantiate(targetClz);
+			target.setConf(config);
+			// run(String...) swallows the exceptions - re-implement it here
+			//
+			LinkedList<String> args = new LinkedList<String>(Arrays.asList(argvs));
+			try {
+				Method m = ReflectionUtils.findMethod(cmd, "processOptions", LinkedList.class);
+				ReflectionUtils.makeAccessible(m);
+				ReflectionUtils.invokeMethod(m, target, args);
+				m = ReflectionUtils.findMethod(cmd, "processRawArguments", LinkedList.class);
+				ReflectionUtils.makeAccessible(m);
+				ReflectionUtils.invokeMethod(m, target, args);
+			} catch (IllegalStateException ex){
+				throw new RuntimeException("Cannot change permissions/ownership " + ex);
+			}
+		}
+	}
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/jsonworkflownodes/PortSpecifications.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/jsonworkflownodes/PortSpecifications.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/jsonworkflownodes/PortSpecifications.java	(revision 51263)
@@ -0,0 +1,66 @@
+package eu.dnetlib.dhp.common.java.jsonworkflownodes;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.avro.Schema;
+
+import eu.dnetlib.dhp.common.java.jsonworkflownodes.StringPortSpecificationExtractor.PortSpecification;
+import eu.dnetlib.dhp.common.java.porttype.AvroPortType;
+import eu.dnetlib.dhp.common.java.porttype.PortType;
+import eu.dnetlib.dhp.common.utils.AvroUtils;
+
+/**
+ * @author Mateusz Kobos
+ */
+public class PortSpecifications {
+	private static final String[] propertyRegexps = 
+			new String[]{"[\\w\\.]+", "[\\w\\./_\\-]+"};
+	private final Map<String, SpecificationValues> specs;
+	
+    public static class SpecificationValues {
+
+        private final Schema schema;
+
+        private final String jsonFilePath;
+
+        public SpecificationValues(Schema schema, String jsonFilePath) {
+            this.schema = schema;
+            this.jsonFilePath = jsonFilePath;
+        }
+
+        public Schema getSchema() {
+            return schema;
+        }
+
+        public String getJsonFilePath() {
+            return jsonFilePath;
+        }
+
+    }
+	
+	public PortSpecifications(String[] portSpecifications){
+		StringPortSpecificationExtractor portSpecExtractor = 
+				new StringPortSpecificationExtractor(propertyRegexps);
+		specs = new HashMap<String, SpecificationValues>();
+		for(int i = 0; i < portSpecifications.length; i++){
+			PortSpecification portSpec = portSpecExtractor.getSpecification(portSpecifications[i]);
+			Schema schema = AvroUtils.toSchema(portSpec.getProperties()[0]);
+			String jsonPath = portSpec.getProperties()[1];
+			specs.put(portSpec.getName(), new SpecificationValues(schema, jsonPath));
+		}
+	}
+	
+	public SpecificationValues get(String portName){
+		return specs.get(portName);
+	}
+	
+	public Map<String, PortType> getPortTypes(){
+		Map<String, PortType> ports = new HashMap<String, PortType>();
+		for(Map.Entry<String, SpecificationValues> e: specs.entrySet()){
+			Schema schema = e.getValue().schema;
+			ports.put(e.getKey(), new AvroPortType(schema));
+		}
+		return ports;	
+	}
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/ProcessException.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/ProcessException.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/ProcessException.java	(revision 51263)
@@ -0,0 +1,20 @@
+package eu.dnetlib.dhp.common.java;
+
+/**
+ * Process exception
+ * @author Dominika Tkaczyk
+ *
+ */
+public class ProcessException extends RuntimeException {
+
+	private static final long serialVersionUID = 2758953138374438377L;
+
+	public ProcessException(String message){
+		super(message);
+	}
+	
+	public ProcessException(String message, Throwable cause){
+		super(message, cause);
+	}
+
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/protobuf/AvroToProtoBufOneToOneMapper.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/protobuf/AvroToProtoBufOneToOneMapper.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/protobuf/AvroToProtoBufOneToOneMapper.java	(revision 51263)
@@ -0,0 +1,62 @@
+package eu.dnetlib.dhp.common.protobuf;
+
+import com.google.protobuf.Message;
+import org.apache.avro.generic.IndexedRecord;
+import org.apache.avro.mapred.AvroKey;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.log4j.Logger;
+
+import java.io.IOException;
+
+/**
+ * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)
+ */
+public class AvroToProtoBufOneToOneMapper<IN extends IndexedRecord, OUT extends Message>
+        extends Mapper<AvroKey<IN>, NullWritable, Text, BytesWritable> {
+    private static final String CONVERTER_CLASS_PROPERTY = "converter_class";
+    private static final Logger log = Logger.getLogger(AvroToProtoBufOneToOneMapper.class);
+
+    private final Text keyWritable = new Text();
+    private final BytesWritable valueWritable = new BytesWritable();
+    private AvroToProtoBufConverter<IN, OUT> converter;
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public void setup(Context context) throws IOException, InterruptedException {
+        Class<?> converterClass = context.getConfiguration().getClass(CONVERTER_CLASS_PROPERTY, null);
+
+        if (converterClass == null) {
+            throw new IOException("Please specify " + CONVERTER_CLASS_PROPERTY);
+        }
+
+        try {
+            converter = (AvroToProtoBufConverter<IN, OUT>) converterClass.newInstance();
+        } catch (ClassCastException e) {
+            throw new IOException(
+                    "Class specified in " + CONVERTER_CLASS_PROPERTY + " doesn't implement AvroToProtoBufConverter", e);
+        } catch (Exception e) {
+            throw new IOException(
+                    "Could not instantiate specified AvroToProtoBufConverter class, " + converterClass, e);
+        }
+    }
+
+    @Override
+    public void map(AvroKey<IN> avro, NullWritable ignore, Context context)
+            throws IOException, InterruptedException {
+        String key = null;
+        try {
+            key = converter.convertIntoKey(avro.datum());
+            keyWritable.set(key);
+
+            byte[] value = converter.convertIntoValue(avro.datum()).toByteArray();
+            valueWritable.set(value, 0, value.length);
+
+            context.write(keyWritable, valueWritable);
+        } catch (Exception e) {
+            log.error("Error" + (key != null ? " while processing  " + key : ""), e);
+        }
+    }
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/spark/pipe/SparkPipeExecutor.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/spark/pipe/SparkPipeExecutor.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/spark/pipe/SparkPipeExecutor.java	(revision 51263)
@@ -0,0 +1,74 @@
+package eu.dnetlib.dhp.common.spark.pipe;
+
+import java.io.Serializable;
+
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.mapred.AvroKey;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.spark.SparkFiles;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaRDD;
+
+import eu.dnetlib.dhp.common.utils.AvroGsonFactory;
+import scala.Tuple2;
+
+
+/**
+ * Executor of mapreduce scripts using spark pipes.
+ * It imitates hadoop streaming behavior. 
+ * 
+ * @author madryk
+ *
+ */
+public class SparkPipeExecutor implements Serializable {
+
+	private static final long serialVersionUID = 1L;
+
+
+	//------------------------ LOGIC --------------------------
+	
+	/**
+	 * Imitates map part of hadoop streaming job.
+	 * It executes provided script for every key in inputRecords rdd.
+	 * <br/><br/>
+	 * It is assumed that provided script will read records from standard input (one line for one record)
+	 * and write mapped record into standard output (also one line for one record).
+	 * Mapped record can be a key/value pair. In that case script should return key and value
+	 * splitted by tab (\t) character in single line. 
+	 */
+	public JavaPairRDD<String, String> doMap(JavaPairRDD<AvroKey<GenericRecord>, NullWritable> inputRecords, String scriptName, String args) {
+
+		JavaRDD<String> mappedRecords = inputRecords.keys().pipe("python " + SparkFiles.get(scriptName) + " " + args);
+
+		JavaPairRDD<String, String> outputRecords = mappedRecords
+				.mapToPair(line -> {
+					String[] splittedPair = line.split("\t");
+					return new Tuple2<String, String>(splittedPair[0], (splittedPair.length == 1) ? null : splittedPair[1]);
+				});
+
+		return outputRecords;
+	}
+
+	/**
+	 * Imitates reduce part of hadoop streaming job.
+	 * <br/><br/>
+	 * It is assumed that provided script will read records from standard input (one line for one record)
+	 * and group records with the same key into single record (reduce).
+	 * Method assures that all input records with the same key will be transfered in adjacent lines.
+	 * Reduced records should be written by script into standard output (one line for one record).
+	 * Reduced records must be json strings of class provided as argument.
+	 */
+	public JavaPairRDD<AvroKey<GenericRecord>, NullWritable> doReduce(JavaPairRDD<String, String> inputRecords, String scriptName, String args, Class<? extends GenericRecord> outputClass) {
+
+		JavaRDD<String> reducedRecords = inputRecords.sortByKey()
+				.map(record -> record._1 + ((record._2 == null) ? "" : ("\t" + record._2)))
+				.pipe("python " + SparkFiles.get(scriptName) + " " + args);
+
+		JavaPairRDD<AvroKey<GenericRecord>, NullWritable> outputRecords = reducedRecords
+				.map(recordString -> AvroGsonFactory.create().fromJson(recordString, outputClass))
+				.mapToPair(record -> new Tuple2<AvroKey<GenericRecord>, NullWritable>(new AvroKey<>(record), NullWritable.get()));
+
+		return outputRecords;
+	}
+
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/EmptyDatastoreVerifierProcess.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/EmptyDatastoreVerifierProcess.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/EmptyDatastoreVerifierProcess.java	(revision 51263)
@@ -0,0 +1,89 @@
+package eu.dnetlib.dhp.common.utils;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.security.InvalidParameterException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import eu.dnetlib.dhp.common.java.PortBindings;
+import eu.dnetlib.dhp.common.java.Ports;
+import eu.dnetlib.dhp.common.java.Process;
+import eu.dnetlib.dhp.common.java.io.CloseableIterator;
+import eu.dnetlib.dhp.common.java.io.DataStore;
+import eu.dnetlib.dhp.common.java.io.FileSystemPath;
+import eu.dnetlib.dhp.common.java.porttype.AnyPortType;
+import eu.dnetlib.dhp.common.java.porttype.PortType;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.OOZIE_ACTION_OUTPUT_FILENAME;
+
+/**
+ * Simple process verifying whether given datastore is empty.
+ * @author mhorst
+ *
+ */
+public class EmptyDatastoreVerifierProcess implements Process {
+
+	public static final String INPUT_PORT_NAME = "input";
+	
+	public static final String DEFAULT_ENCODING = "UTF-8";
+	
+	public static final String OUTPUT_PROPERTY_IS_EMPTY = "isEmpty";
+
+    /**
+     * Ports handled by this module.
+     */
+    private final Ports ports;
+
+	
+	// ------------------------ CONSTRUCTORS --------------------------
+	
+	public EmptyDatastoreVerifierProcess() {
+//      preparing ports
+        Map<String, PortType> input = new HashMap<String, PortType>();
+        input.put(INPUT_PORT_NAME, new AnyPortType());
+        Map<String, PortType> output = Collections.emptyMap();
+        ports = new Ports(input, output); 
+	}
+	
+	@Override
+	public Map<String, PortType> getInputPorts() {
+		return ports.getInput();
+	}
+
+	@Override
+	public Map<String, PortType> getOutputPorts() {
+		return ports.getOutput();
+	}
+
+	@Override
+	public void run(PortBindings portBindings, Configuration conf, Map<String, String> parameters) throws Exception {
+	    if (!portBindings.getInput().containsKey(INPUT_PORT_NAME)) {
+	        throw new InvalidParameterException("missing input port!");
+	    }
+	    
+		try (CloseableIterator<?> closeableIt = getIterator(conf, portBindings.getInput().get(INPUT_PORT_NAME))) {
+			File file = new File(System.getProperty(OOZIE_ACTION_OUTPUT_FILENAME));
+	        Properties props = new Properties();
+			props.setProperty(OUTPUT_PROPERTY_IS_EMPTY, Boolean.toString(!closeableIt.hasNext()));	
+	        try (OutputStream os = new FileOutputStream(file)) {
+		        	props.store(os, "");	
+	        }
+		}
+	}
+	
+	/**
+	 * Returns iterator over datastore.
+	 */
+	protected CloseableIterator<?> getIterator(Configuration conf, Path path) throws IOException {
+	    return DataStore.getReader(new FileSystemPath(FileSystem.get(conf), path));
+	}
+
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/ByteArrayUtils.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/ByteArrayUtils.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/ByteArrayUtils.java	(revision 51263)
@@ -0,0 +1,45 @@
+package eu.dnetlib.dhp.common.utils;
+
+/**
+ * Byte array utility class.
+ * @author mhorst
+ *
+ */
+public final class ByteArrayUtils {
+    
+    //------------------------ CONSTRUCTORS -------------------
+
+    private ByteArrayUtils() {}
+    
+    //------------------------ LOGIC --------------------------
+    
+	/**
+	 * Does this byte array begin with match array content?
+	 * @param source Byte array to examine
+	 * @param match Byte array to locate in <code>source</code>
+	 * @return true If the starting bytes are equal
+	 */
+	public static boolean startsWith(byte[] source, byte[] match) {
+		return startsWith(source, 0, match);
+	}
+
+	/**
+	 * Does this byte array begin with match array content?
+	 * @param source Byte array to examine
+	 * @param offset An offset into the <code>source</code> array
+	 * @param match Byte array to locate in <code>source</code>
+	 * @return true If the starting bytes are equal
+	 */
+	public static boolean startsWith(byte[] source, int offset, byte[] match) {
+		if (match.length > (source.length - offset)) {
+			return false;
+		}
+		for (int i = 0; i < match.length; i++) {
+			if (source[offset + i] != match[i]) {
+				return false;
+			}
+		}
+		return true;
+	}
+	
+}
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MongoRecordImporter.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MongoRecordImporter.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MongoRecordImporter.java	(revision 51263)
@@ -0,0 +1,48 @@
+package eu.dnetlib.dhp.wf.importer.mdrecord;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import eu.dnetlib.dhp.common.java.PortBindings;
+import eu.dnetlib.dhp.common.java.Process;
+import eu.dnetlib.dhp.common.java.porttype.PortType;
+import org.apache.hadoop.conf.Configuration;
+
+public class MongoRecordImporter implements Process {
+
+	private final Map<String, PortType> outputPorts = new HashMap<String, PortType>();
+
+	@Override
+	public Map<String, PortType> getInputPorts() {
+		return Collections.emptyMap();
+	}
+
+	@Override
+	public Map<String, PortType> getOutputPorts() {
+		return outputPorts;
+	}
+
+	@Override
+	public void run(final PortBindings portBindings, final Configuration conf, final Map<String, String> parameters) throws Exception {
+
+		/*
+		SparkSession spark = SparkSession.builder()
+				.master("local")
+				.appName("MongoSparkConnectorIntro")
+				.config("spark.mongodb.input.uri", "mongodb://127.0.0.1/test.myCollection")
+				.config("spark.mongodb.output.uri", "mongodb://127.0.0.1/test.myCollection")
+				.getOrCreate();
+
+		// Create a JavaSparkContext using the SparkSession's SparkContext object
+		JavaSparkContext jsc = new JavaSparkContext(spark.sparkContext());
+
+		// More application logic would go here...
+
+		jsc.close();
+		*/
+
+	}
+
+
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/AvroGsonFactory.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/AvroGsonFactory.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/AvroGsonFactory.java	(revision 51263)
@@ -0,0 +1,45 @@
+package eu.dnetlib.dhp.common.utils;
+
+import java.lang.reflect.Type;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonParseException;
+
+/**
+ * Factory for gson object that supports serializing avro generated classes
+ * 
+ * @author madryk
+ *
+ */
+public final class AvroGsonFactory {
+    
+    //------------------------ CONSTRUCTORS -------------------
+    
+    
+    private AvroGsonFactory() {}
+
+	
+	//------------------------ LOGIC --------------------------
+	
+	public static Gson create() {
+		GsonBuilder builder = new GsonBuilder();
+
+		builder.registerTypeAdapter(CharSequence.class, new CharSequenceDeserializer());
+
+		return builder.create();
+	}
+
+	public static class CharSequenceDeserializer implements JsonDeserializer<CharSequence> {
+
+		@Override
+		public CharSequence deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context)
+				throws JsonParseException {
+			return json.getAsString();
+		}
+
+	}
+}
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacadeFactory.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacadeFactory.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacadeFactory.java	(revision 51263)
@@ -0,0 +1,45 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_CLIENT_CONNECTION_TIMEOUT;
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_CLIENT_READ_TIMEOUT;
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_CONNECTION_TIMEOUT_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_READ_TIMEOUT_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_ISLOOKUP_SERVICE_LOCATION;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_RESULT_SET_CLIENT_CONNECTION_TIMEOUT;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_RESULT_SET_CLIENT_READ_TIMEOUT;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_RESULT_SET_PAGESIZE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.RESULTSET_CONNECTION_TIMEOUT_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.RESULTSET_PAGESIZE_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.RESULTSET_READ_TIMEOUT_DEFAULT_VALUE;
+
+import java.util.Map;
+
+import com.google.common.base.Preconditions;
+
+import eu.dnetlib.dhp.common.WorkflowRuntimeParameters;
+
+/**
+ * WebService Database service facade factory.
+ * 
+ * @author mhorst
+ *
+ */
+public class WebServiceISLookupFacadeFactory implements ServiceFacadeFactory<ISLookupFacade> {
+
+    
+    //------------------------ LOGIC --------------------------
+    
+    @Override
+    public ISLookupFacade instantiate(Map<String, String> parameters) {
+        Preconditions.checkArgument(parameters.containsKey(IMPORT_ISLOOKUP_SERVICE_LOCATION), 
+                "unknown ISLookup service location: no parameter provided:  '%s'", IMPORT_ISLOOKUP_SERVICE_LOCATION);
+        
+        return new WebServiceISLookupFacade(parameters.get(IMPORT_ISLOOKUP_SERVICE_LOCATION),
+                Long.parseLong(WorkflowRuntimeParameters.getParamValue(DNET_SERVICE_CLIENT_READ_TIMEOUT, DNET_SERVICE_READ_TIMEOUT_DEFAULT_VALUE, parameters)),
+                Long.parseLong(WorkflowRuntimeParameters.getParamValue(DNET_SERVICE_CLIENT_CONNECTION_TIMEOUT, DNET_SERVICE_CONNECTION_TIMEOUT_DEFAULT_VALUE, parameters)),
+                Long.parseLong(WorkflowRuntimeParameters.getParamValue(IMPORT_RESULT_SET_CLIENT_READ_TIMEOUT, RESULTSET_READ_TIMEOUT_DEFAULT_VALUE, parameters)),
+                Long.parseLong(WorkflowRuntimeParameters.getParamValue(IMPORT_RESULT_SET_CLIENT_CONNECTION_TIMEOUT, RESULTSET_CONNECTION_TIMEOUT_DEFAULT_VALUE, parameters)),
+                Integer.parseInt(WorkflowRuntimeParameters.getParamValue(IMPORT_RESULT_SET_PAGESIZE, RESULTSET_PAGESIZE_DEFAULT_VALUE, parameters)));
+    }
+
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/AvroUtils.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/AvroUtils.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/utils/AvroUtils.java	(revision 51263)
@@ -0,0 +1,77 @@
+package eu.dnetlib.dhp.common.utils;
+
+import java.lang.reflect.Field;
+
+import org.apache.avro.Schema;
+
+/**
+ * 
+ * @author Mateusz Kobos
+ *
+ */
+public final class AvroUtils {
+    
+	public final static String primitiveTypePrefix = "org.apache.avro.Schema.Type.";
+	
+	
+	//------------------------ CONSTRUCTORS -------------------
+	
+	
+	private AvroUtils() {}
+	
+	
+	//------------------------ LOGIC --------------------------
+	
+	
+	/**
+	 * For a given name of a class generated from Avro schema return 
+	 * a JSON schema.
+	 * 
+	 * Apart from a name of a class you can also give a name of one of enums
+	 * defined in {@link org.apache.avro.Schema.Type}; in such case an
+	 * appropriate primitive type will be returned.
+	 * 
+	 * @param typeName fully qualified name of a class generated from Avro schema, 
+	 * e.g. {@code eu.dnetlib.dhp.common.avro.Person},
+	 * or a fully qualified name of enum defined by 
+	 * {@link org.apache.avro.Schema.Type},
+	 * e.g. {@link org.apache.avro.Schema.Type.STRING}.
+	 * @return JSON string
+	 */
+	public static Schema toSchema(String typeName) {
+		Schema schema = null;
+		if(typeName.startsWith(primitiveTypePrefix)){
+			String shortName = typeName.substring(
+					primitiveTypePrefix.length(), typeName.length());
+			schema = getPrimitiveTypeSchema(shortName);
+		} else {
+			schema = getAvroClassSchema(typeName);
+		}
+		return schema;
+	}
+	
+	private static Schema getPrimitiveTypeSchema(String shortName){
+		Schema.Type type = Schema.Type.valueOf(shortName);
+		return Schema.create(type);
+	}
+	
+	private static Schema getAvroClassSchema(String className){
+		try {
+			Class<?> avroClass = Class.forName(className);
+			Field f = avroClass.getDeclaredField("SCHEMA$");
+			return (Schema) f.get(null);
+		} catch (ClassNotFoundException e) {
+			throw new RuntimeException(
+					"Class \""+className+"\" does not exist", e);
+		} catch (SecurityException e) {
+			throw new RuntimeException(e);
+		} catch (NoSuchFieldException e) {
+			throw new RuntimeException(e);
+		} catch (IllegalArgumentException e) {
+			throw new RuntimeException(e);
+		} catch (IllegalAccessException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/ProcessUtils.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/ProcessUtils.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/ProcessUtils.java	(revision 51263)
@@ -0,0 +1,43 @@
+package eu.dnetlib.dhp.common.java;
+
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * {@link Process} related utility class.
+ * @author mhorst
+ *
+ */
+public final class ProcessUtils {
+
+    // ------------- CONSTRUCTORS ----------------
+    
+    private ProcessUtils() {}
+    
+    // ------------- LOGIC -----------------------
+    
+	/**
+	 * Returns parameter value retrived from parameters or context.
+	 * @param paramName
+	 * @param hadoopConf
+	 * @param parameters
+	 * @return parameter value
+	 */
+	public static String getParameterValue(String paramName, 
+			Configuration hadoopConf,
+			Map<String, String> parameters) {
+		if (parameters!=null && !parameters.isEmpty()) {
+			String result = null;
+			result = parameters.get(paramName);
+			if (result!=null) {
+				return result;
+			}
+		}
+		if (hadoopConf!=null) {
+			return hadoopConf.get(paramName);	
+		} else {
+			return null;
+		}
+	}
+}
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeUtils.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeUtils.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeUtils.java	(revision 51263)
@@ -0,0 +1,80 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_FACADE_FACTORY_CLASS;
+
+import java.lang.reflect.Constructor;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.collect.ImmutableMap;
+
+import eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters;
+
+/**
+ * Service facade utility methods.
+ * @author mhorst
+ *
+ */
+public final class ServiceFacadeUtils {
+
+    //------------------------ CONSTRUCTORS -------------------
+    
+    private ServiceFacadeUtils() {}
+    
+    //------------------------ LOGIC --------------------------
+    
+    /**
+     * Instantiates service based on provided parameters.
+     * 
+     * Service factory class name is mandatory and has to be provided as {@value ImportWorkflowRuntimeParameters#IMPORT_FACADE_FACTORY_CLASS} parameter.
+     * Other parameters will be used by factory itself. Factory must be instantiable with no-argument construtor.
+     * 
+     * @param parameters set of parameters required for service instantiation
+     * 
+     */
+    public static <T> T instantiate(Map<String, String> parameters) throws ServiceFacadeException {
+        String serviceFactoryClassName = parameters.get(IMPORT_FACADE_FACTORY_CLASS);
+        if (StringUtils.isBlank(serviceFactoryClassName)) {
+            throw new ServiceFacadeException("unknown service facade factory, no " + IMPORT_FACADE_FACTORY_CLASS + " parameter provided!");
+        }
+        try {
+            Class<?> clazz = Class.forName(serviceFactoryClassName);
+            Constructor<?> constructor = clazz.getConstructor();
+            @SuppressWarnings("unchecked")
+            ServiceFacadeFactory<T> serviceFactory = (ServiceFacadeFactory<T>) constructor.newInstance();
+            return serviceFactory.instantiate(parameters);    
+        } catch (Exception e) {
+            throw new ServiceFacadeException("exception occurred while instantiating service by facade factory: " + IMPORT_FACADE_FACTORY_CLASS, e);
+        }
+        
+    }
+    
+    /**
+     * Instantiates service based on provided configuration.
+     * 
+     * Service factory class name is mandatory and has to be provided as {@value ImportWorkflowRuntimeParameters#IMPORT_FACADE_FACTORY_CLASS} configuration entry.
+     * Other parameters will be used by factory itself. Factory must be instantiable with no-argument construtor.
+     * 
+     * @param config set of configuration entries required for service instantiation
+     */
+    public static <T> T instantiate(Configuration config) throws ServiceFacadeException {
+        return instantiate(buildParameters(config));
+    }
+    
+
+    // ------------------------ PRIVATE --------------------------
+    
+    /**
+     * Converts configuration entries into plain map.
+     */
+    private static Map<String, String> buildParameters(Configuration config) {
+        ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+        for (Map.Entry<String, String> entry : config) {
+          builder.put(entry);
+        }
+        return builder.build();
+    }
+    
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/ProcessWrapper.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/ProcessWrapper.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/ProcessWrapper.java	(revision 51263)
@@ -0,0 +1,88 @@
+package eu.dnetlib.dhp.common.java;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericContainer;
+import org.apache.commons.cli.CommandLine;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+
+import eu.dnetlib.dhp.common.java.io.DataStore;
+import eu.dnetlib.dhp.common.java.io.FileSystemPath;
+import eu.dnetlib.dhp.common.java.porttype.AvroPortType;
+import eu.dnetlib.dhp.common.java.porttype.PortType;
+
+/**
+ * Creates {@link Process} object through reflection by parsing 
+ * the command-line arguments
+ * @author Mateusz Kobos
+ *
+ */
+public class ProcessWrapper {
+	
+	public Configuration getConfiguration() throws Exception{
+		return new Configuration();
+	}
+	
+	public static void main(String[] args) throws Exception {
+		ProcessWrapper wrapper = new ProcessWrapper();
+		wrapper.run(args);
+	}
+	
+	public void run(String[] args) throws Exception{
+		CommandLine cmdLine = CmdLineParser.parse(args);
+		
+		CmdLineParserForProcessConstruction constructionParser = 
+				new CmdLineParserForProcessConstruction();
+		Process process = constructionParser.run(cmdLine);
+		Ports ports = 
+				new Ports(process.getInputPorts(), process.getOutputPorts());
+		CmdLineParserForProcessRunParameters runParametersParser =
+				new CmdLineParserForProcessRunParameters();
+		ProcessParameters params = runParametersParser.run(cmdLine, ports);
+		Configuration conf = getConfiguration();
+		process.run(params.getPortBindings(), conf, params.getParameters());
+		createOutputsIfDontExist(
+				process.getOutputPorts(), params.getPortBindings().getOutput(),
+				conf);
+	}
+	
+	private static void createOutputsIfDontExist(
+			Map<String, PortType> outputPortsSpecification, 
+			Map<String, Path> outputPortBindings, Configuration conf) throws IOException{
+		FileSystem fs = FileSystem.get(conf);
+		for(Map.Entry<String, Path> entry: outputPortBindings.entrySet()){
+			Path path = entry.getValue();
+			if(!fs.exists(path) || isEmptyDirectory(fs, path)){
+				PortType rawType = outputPortsSpecification.get(entry.getKey());
+				if(!(rawType instanceof AvroPortType)){
+					throw new RuntimeException("The port \""+entry.getKey()+
+							"\" is not of Avro type and only Avro types are "+
+							"supported");
+				}
+				AvroPortType type = (AvroPortType) rawType;
+				FileSystemPath fsPath = new FileSystemPath(fs, path);
+				DataFileWriter<GenericContainer> writer = 
+						DataStore.create(fsPath, type.getSchema());
+				writer.close();
+			}
+		}
+	}
+	
+	private static boolean isEmptyDirectory(FileSystem fs, Path path) throws IOException{
+		if(!fs.isDirectory(path)){
+			return false;
+		}
+		RemoteIterator<LocatedFileStatus> files = fs.listFiles(path, false);
+		/** There's at least one file, so the directory is not empty */
+		if(files.hasNext()){
+			return false;
+		}
+		return true;
+	}
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/ProcessParameters.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/ProcessParameters.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/ProcessParameters.java	(revision 51263)
@@ -0,0 +1,43 @@
+package eu.dnetlib.dhp.common.java;
+
+import java.util.Map;
+
+import org.apache.commons.lang.NotImplementedException;
+
+/**
+ * Parameters of the Process retrieved from Oozie
+ * @author Mateusz Kobos
+ *
+ */
+public class ProcessParameters {
+	private final PortBindings portBindings;
+	private final Map<String, String> parameters;
+	
+	public PortBindings getPortBindings() {
+		return portBindings;
+	}
+	
+	public Map<String, String> getParameters(){
+		return parameters;
+	}
+
+	public ProcessParameters(PortBindings portBindings, 
+			Map<String, String> parameters) {
+		this.portBindings = portBindings;
+		this.parameters =  parameters;
+	}
+	
+	@Override
+	public boolean equals(Object o){
+		if(!(o instanceof ProcessParameters)){
+			return false;
+		}
+		ProcessParameters other = (ProcessParameters) o;
+		return this.portBindings.equals(other.portBindings);
+	}
+	
+	@Override
+	public int hashCode(){
+		throw new NotImplementedException();
+	}
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/InfoSpaceConstants.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/InfoSpaceConstants.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/InfoSpaceConstants.java	(revision 51263)
@@ -0,0 +1,75 @@
+package eu.dnetlib.dhp.common;
+
+import java.io.UnsupportedEncodingException;
+
+/**
+ * InfoSpaceConstants constants.
+ * 
+ * @author mhorst
+ *
+ */
+public final class InfoSpaceConstants {
+
+    public static final float CONFIDENCE_TO_TRUST_LEVEL_FACTOR = 0.9f;
+
+    public static final String ENCODING_UTF8 = "utf-8";
+
+    public static final char ROW_PREFIX_SEPARATOR = '|';
+
+    public static final String ID_NAMESPACE_SEPARATOR = "::";
+    public static final String CLASSIFICATION_HIERARCHY_SEPARATOR = ID_NAMESPACE_SEPARATOR;
+    public static final String INFERENCE_PROVENANCE_SEPARATOR = ID_NAMESPACE_SEPARATOR;
+
+    public static final String ROW_PREFIX_RESULT = "50|";
+    public static final String ROW_PREFIX_PROJECT = "40|";
+    public static final String ROW_PREFIX_PERSON = "30|";
+    public static final String ROW_PREFIX_ORGANIZATION = "20|";
+    public static final String ROW_PREFIX_DATASOURCE = "10|";
+
+    public static final String QUALIFIER_BODY_STRING = "body";
+    public static final byte[] QUALIFIER_BODY;
+
+    public static final String SEMANTIC_CLASS_MAIN_TITLE = "main title";
+    public static final String SEMANTIC_CLASS_PUBLICATION = "publication";
+    public static final String SEMANTIC_CLASS_UNKNOWN = "UNKNOWN";
+
+    public static final String SEMANTIC_SCHEME_DNET_PERSON_ROLES = "dnet:personroles";
+    public static final String SEMANTIC_SCHEME_DNET_RELATIONS_RESULT_RESULT = "dnet:result_result_relations";
+    public static final String SEMANTIC_SCHEME_DNET_RELATIONS_RESULT_PROJECT = "dnet:result_project_relations";
+
+    public static final String SEMANTIC_SCHEME_DNET_TITLE = "dnet:dataCite_title";
+    public static final String SEMANTIC_SCHEME_DNET_TITLE_TYPOLOGIES = "dnet:title_typologies";
+    public static final String SEMANTIC_SCHEME_DNET_RESULT_TYPOLOGIES = "dnet:result_typologies";
+    public static final String SEMANTIC_SCHEME_DNET_PROVENANCE_ACTIONS = "dnet:provenanceActions";
+    public static final String SEMANTIC_SCHEME_DNET_LANGUAGES = "dnet:languages";
+    public static final String SEMANTIC_SCHEME_DNET_PID_TYPES = "dnet:pid_types";
+    public static final String SEMANTIC_SCHEME_DNET_CLASSIFICATION_TAXONOMIES = "dnet:subject_classification_typologies";
+
+    // resultResult citation and similarity related
+    public static final String SEMANTIC_SCHEME_DNET_DATASET_PUBLICATION_RELS = "dnet:dataset_publication_rels";
+
+    public static final String SEMANTIC_CLASS_TAXONOMIES_ARXIV = "arxiv";
+    public static final String SEMANTIC_CLASS_TAXONOMIES_WOS = "wos";
+    public static final String SEMANTIC_CLASS_TAXONOMIES_DDC = "ddc";
+    public static final String SEMANTIC_CLASS_TAXONOMIES_MESHEUROPMC = "mesheuropmc";
+    public static final String SEMANTIC_CLASS_TAXONOMIES_ACM = "acm";
+
+    public static final String EXTERNAL_ID_TYPE_INSTANCE_URL = "dnet:instance-url";
+    public static final String EXTERNAL_ID_TYPE_UNKNOWN = "unknown";
+
+    // publication types class ids
+    public static final String SEMANTIC_CLASS_INSTANCE_TYPE_ARTICLE = "0001";
+    public static final String SEMANTIC_CLASS_INSTANCE_TYPE_DATASET = "0021";
+
+    static {
+        try {
+            QUALIFIER_BODY = QUALIFIER_BODY_STRING.getBytes(ENCODING_UTF8);
+
+        } catch (UnsupportedEncodingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private InfoSpaceConstants() {
+    }
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/PortBindings.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/PortBindings.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/PortBindings.java	(revision 51263)
@@ -0,0 +1,43 @@
+package eu.dnetlib.dhp.common.java;
+
+import java.util.Map;
+
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Port names (see {@link Ports}) bound to certain paths in the file system
+ * @author Mateusz Kobos
+ *
+ */
+public class PortBindings {
+	private final Map<String, Path> input;
+	private final Map<String, Path> output;
+
+	public PortBindings(Map<String, Path> input, Map<String, Path> output) {
+		this.input = input;
+		this.output = output;
+	}
+	
+	public Map<String, Path> getInput() {
+		return input;
+	}
+	
+	public Map<String, Path> getOutput() {
+		return output;
+	}
+	
+	@Override
+	public boolean equals(Object o){
+		if(!(o instanceof PortBindings)){
+			return false;
+		}
+		PortBindings other = (PortBindings) o;
+		return input.equals(other.input) && output.equals(other.output);
+	}
+	
+	@Override
+	public int hashCode(){
+		throw new NotImplementedException();
+	}
+}
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacadeFactory.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacadeFactory.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacadeFactory.java	(revision 51263)
@@ -0,0 +1,45 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_CLIENT_CONNECTION_TIMEOUT;
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_CLIENT_READ_TIMEOUT;
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_CONNECTION_TIMEOUT_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_READ_TIMEOUT_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_MDSTORE_SERVICE_LOCATION;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_RESULT_SET_CLIENT_CONNECTION_TIMEOUT;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_RESULT_SET_CLIENT_READ_TIMEOUT;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_RESULT_SET_PAGESIZE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.RESULTSET_CONNECTION_TIMEOUT_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.RESULTSET_PAGESIZE_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.RESULTSET_READ_TIMEOUT_DEFAULT_VALUE;
+
+import java.util.Map;
+
+import com.google.common.base.Preconditions;
+
+import eu.dnetlib.dhp.common.WorkflowRuntimeParameters;
+
+/**
+ * WebService MDStore service facade factory.
+ * 
+ * @author mhorst
+ *
+ */
+public class WebServiceMDStoreFacadeFactory implements ServiceFacadeFactory<MDStoreFacade> {
+
+
+    //------------------------ LOGIC --------------------------
+    
+    @Override
+    public WebServiceMDStoreFacade instantiate(Map<String, String> parameters) {
+        Preconditions.checkArgument(parameters.containsKey(IMPORT_MDSTORE_SERVICE_LOCATION), 
+                "unknown MDStore service location: no parameter provided:  '%s'", IMPORT_MDSTORE_SERVICE_LOCATION);
+
+        return new WebServiceMDStoreFacade(parameters.get(IMPORT_MDSTORE_SERVICE_LOCATION),
+                Long.parseLong(WorkflowRuntimeParameters.getParamValue(DNET_SERVICE_CLIENT_READ_TIMEOUT, DNET_SERVICE_READ_TIMEOUT_DEFAULT_VALUE, parameters)),
+                Long.parseLong(WorkflowRuntimeParameters.getParamValue(DNET_SERVICE_CLIENT_CONNECTION_TIMEOUT, DNET_SERVICE_CONNECTION_TIMEOUT_DEFAULT_VALUE, parameters)),
+                Long.parseLong(WorkflowRuntimeParameters.getParamValue(IMPORT_RESULT_SET_CLIENT_READ_TIMEOUT, RESULTSET_READ_TIMEOUT_DEFAULT_VALUE, parameters)),
+                Long.parseLong(WorkflowRuntimeParameters.getParamValue(IMPORT_RESULT_SET_CLIENT_CONNECTION_TIMEOUT, RESULTSET_CONNECTION_TIMEOUT_DEFAULT_VALUE, parameters)),
+                Integer.parseInt(WorkflowRuntimeParameters.getParamValue(IMPORT_RESULT_SET_PAGESIZE, RESULTSET_PAGESIZE_DEFAULT_VALUE, parameters)));
+    }
+
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/porttype/PortType.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/porttype/PortType.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/porttype/PortType.java	(revision 51263)
@@ -0,0 +1,33 @@
+package eu.dnetlib.dhp.common.java.porttype;
+
+/**
+ * Type of the port. This is used to specify what kind of data is 
+ * accepted on a certain input port or produced on a certain output port 
+ * of a workflow node.
+ * 
+ * @author Mateusz Kobos
+ *
+ */
+public interface PortType {
+	
+	String getName();
+	
+	/**
+	 * This should be used to check whether data produced by a workflow node
+	 * conforms to the data consumed by other workflow node. 
+	 * In a scenario when A produces certain data on a port p and B consumes 
+	 * this data on a port q, type(q).accepts(type(p)) has to be true.
+	 * 
+	 * @return {@code true} if {@code this} port type is a more general 
+	 * version of the {@code other} port type, 
+	 * or as an alternative explanation: {@code other} is a subset of 
+	 * {@code this}, i.e. {@code other} has at least all the properties present 
+	 * in {@code this} (and possibly some others). This is analogous to a 
+	 * situation in object-oriented programming, where in order for assignment 
+	 * operation {@code this = other} to work, the type of {@code this} has to 
+	 * accept type of {@code other}, or in other words {@code other} has to 
+	 * inherit from {@code this}, or in yet other words: {@code other} has to
+	 * conform to {@code this}.
+	 */
+	boolean accepts(PortType other);
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/porttype/AvroPortType.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/porttype/AvroPortType.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/porttype/AvroPortType.java	(revision 51263)
@@ -0,0 +1,65 @@
+package eu.dnetlib.dhp.common.java.porttype;
+
+import org.apache.avro.Schema;
+import org.apache.commons.lang.NotImplementedException;
+
+/**
+ * This port type accepts data stores in a format of Avro 
+ * Object Container Files, i.e. Avro data files. 
+ * This kind of file corresponds to a list of objects, each one being of the
+ * same type, i.e. each one is defined by the same Avro schema.
+ * @author Mateusz Kobos
+ */
+public class AvroPortType implements PortType {
+	
+	private final Schema schema;
+	
+
+	public AvroPortType(Schema schema) {
+		this.schema = schema;
+	}
+
+	@Override
+	public String getName() {
+		return schema.getFullName();
+	}
+
+	@Override
+	/** Simple check if the port types are exactly the same 
+	 * (as defined by the {@code equals} method). 
+	 * 
+	 * TODO: this should work in a more relaxed way - 
+	 * {@code this.accepts(other)} should be true if {@code this} 
+	 * describes a subset of structures defined in {@code other}. To be
+	 * more precise: the JSON schema tree tree defined by {@code this} should
+	 * form a sub-tree of the JSON schema tree defined by {@code other}. */
+	public boolean accepts(PortType other) {
+		return this.equals(other);
+	}
+	
+	/**
+	 * Two patterns are equal if their schemas are the same.
+	 */
+	@Override
+	public boolean equals(Object o){
+		if(!(o instanceof AvroPortType)){
+			return false;
+		}
+		AvroPortType other = (AvroPortType) o;
+		return this.schema.equals(other.schema);
+	}
+	
+	@Override
+	public int hashCode(){
+		throw new NotImplementedException();
+	}
+
+	/**
+	 * Returns avro schema.
+	 * @return avro schema
+	 */
+	public Schema getSchema() {
+		return schema;
+	}
+
+}
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/RecordReceiver.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/RecordReceiver.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/RecordReceiver.java	(revision 51263)
@@ -0,0 +1,14 @@
+package eu.dnetlib.dhp.wf.importer;
+
+import java.io.IOException;
+
+/**
+ * Record receiver interface.
+ * @author mhorst
+ *
+ * @param <T>
+ */
+public interface RecordReceiver<T> {
+
+	void receive(T object) throws IOException; 
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/porttype/AnyPortType.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/porttype/AnyPortType.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/porttype/AnyPortType.java	(revision 51263)
@@ -0,0 +1,20 @@
+package eu.dnetlib.dhp.common.java.porttype;
+
+/**
+ * A port type that accepts any type of data
+ * @author Mateusz Kobos
+ *
+ */
+public class AnyPortType implements PortType {
+
+	@Override
+	public String getName() {
+		return "Any";
+	}
+
+	@Override
+	public boolean accepts(PortType other) {
+		return true;
+	}
+
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/CountingIterator.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/CountingIterator.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/CountingIterator.java	(revision 51263)
@@ -0,0 +1,19 @@
+package eu.dnetlib.dhp.common.java.io;
+
+import java.util.Iterator;
+
+/**
+ * Counting iterator providing total number of results.
+ * @author mhorst
+ *
+ * @param <E>
+ */
+public interface CountingIterator<E> extends Iterator<E> {
+
+	/**
+	 * Provides total number of results to be iterating on.
+	 * @return total number of results to be iterating on
+	 */
+	int getCount();
+	
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/fault/FaultUtils.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/fault/FaultUtils.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/fault/FaultUtils.java	(revision 51263)
@@ -0,0 +1,67 @@
+package eu.dnetlib.dhp.common.fault;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import eu.dnetlib.dhp.audit.schemas.Cause;
+import eu.dnetlib.dhp.audit.schemas.Fault;
+
+/**
+ * {@link Fault} related utilities.
+ * @author mhorst
+ *
+ */
+public final class FaultUtils {
+
+    // ---------------------- CONSTRUCTORS -------------------
+    
+    private FaultUtils() {}
+    
+    // ---------------------- LOGIC --------------------------
+    
+	/**
+	 * Generates {@link Fault} instance based on {@link Throwable}.
+	 * @param entityId entity identifier
+	 * @param throwable 
+	 * @param auditSupplementaryData
+	 * @return {@link Fault} instance generated for {@link Throwable}
+	 */
+	public static Fault exceptionToFault(CharSequence entityId, Throwable throwable,
+			Map<CharSequence, CharSequence> auditSupplementaryData) {
+		Fault.Builder faultBuilder = Fault.newBuilder();
+		faultBuilder.setInputObjectId(entityId);
+		faultBuilder.setTimestamp(System.currentTimeMillis());
+		faultBuilder.setCode(throwable.getClass().getName());
+		faultBuilder.setMessage(throwable.getMessage());
+		StringWriter strWriter = new StringWriter();
+		PrintWriter pw = new PrintWriter(strWriter);
+		throwable.printStackTrace(pw);
+		pw.close();
+		faultBuilder.setStackTrace(strWriter.toString());
+		if (throwable.getCause()!=null) {
+			faultBuilder.setCauses(appendThrowableToCauses(
+					throwable.getCause(), new ArrayList<Cause>()));
+		}
+		if (auditSupplementaryData!=null && !auditSupplementaryData.isEmpty()) {
+			faultBuilder.setSupplementaryData(auditSupplementaryData);	
+		}
+		return faultBuilder.build();
+	}
+	
+	protected static List<Cause> appendThrowableToCauses(Throwable e, List<Cause> causes) {
+		Cause.Builder causeBuilder = Cause.newBuilder();
+		causeBuilder.setCode(e.getClass().getName());
+		causeBuilder.setMessage(e.getMessage());
+		causes.add(causeBuilder.build());
+		if (e.getCause()!=null) {
+			return appendThrowableToCauses(
+					e.getCause(),causes);
+		} else {
+			return causes;	
+		}
+	}
+	
+}
Index: dnet-hadoop/dhp-common/pom.xml
===================================================================
--- dnet-hadoop/dhp-common/pom.xml	(nonexistent)
+++ dnet-hadoop/dhp-common/pom.xml	(revision 51263)
@@ -0,0 +1,177 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>eu.dnetlib.dhp</groupId>
+        <artifactId>dhp</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>dhp-common</artifactId>
+    <packaging>jar</packaging>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>dhp-schemas</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.oozie</groupId>
+            <artifactId>oozie-core</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-mapreduce-client-core</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-core_2.10</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_2.10</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.avro</groupId>
+            <artifactId>avro</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.avro</groupId>
+            <artifactId>avro-mapred</artifactId>
+            <classifier>hadoop2</classifier>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
+
+        <dependency>
+            <!-- required by caching mechanism for setting chmod -->
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-beans</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.beust</groupId>
+            <artifactId>jcommander</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.pig</groupId>
+            <artifactId>pig</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.linkedin.datafu</groupId>
+            <artifactId>datafu</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>commons-beanutils</groupId>
+            <artifactId>commons-beanutils</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.jdom</groupId>
+            <artifactId>jdom</artifactId>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>net.alchim31.maven</groupId>
+                <artifactId>scala-maven-plugin</artifactId>
+            </plugin>
+
+            <!-- Plugin that generates Java classes from Avro schemas -->
+            <plugin>
+                <groupId>org.apache.avro</groupId>
+                <artifactId>avro-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>generate-test-sources</phase>
+                        <goals>
+                            <goal>schema</goal>
+                            <goal>idl-protocol</goal>
+                        </goals>
+                        <configuration>
+                            <stringType>String</stringType>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>build-helper-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>add-test-sources</id>
+                        <phase>generate-test-sources</phase>
+                        <goals>
+                            <goal>add-test-source</goal>
+                        </goals>
+                        <configuration>
+                            <sources>
+                                <source>${project.build.directory}/generated-test-sources/avro/</source>
+                            </sources>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <!-- Plugin that generates jar with test classes -->
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <excludedGroups>eu.dnetlib.iis.common.IntegrationTest</excludedGroups>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-failsafe-plugin</artifactId>
+            </plugin>
+
+        </plugins>
+    </build>
+    
+</project>
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/LenientComparisonStringNormalizer.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/LenientComparisonStringNormalizer.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/LenientComparisonStringNormalizer.java	(revision 51263)
@@ -0,0 +1,130 @@
+/*
+ * This file is part of CoAnSys project.
+ * Copyright (c) 2012-2015 ICM-UW
+ * 
+ * CoAnSys is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+
+ * CoAnSys is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with CoAnSys. If not, see <http://www.gnu.org/licenses/>.
+ */
+package eu.dnetlib.dhp.common.string;
+
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * An implementation of {@link StringNormalizer} that normalizes strings for non-strict comparisons
+ * in which one does not care about characters other than letters and digits or about differently written diacritics.
+ *
+ * @author Łukasz Dumiszewski
+ *
+ */
+public final class LenientComparisonStringNormalizer implements StringNormalizer, Serializable {
+
+
+    private static final long serialVersionUID = 1L;
+    
+    
+    private List<Character> whitelistCharacters;
+    
+    
+    //------------------------ CONSTRUCTORS --------------------------
+    
+    public LenientComparisonStringNormalizer() {
+        this(ImmutableList.of());
+    }
+    
+    /**
+     * @param whitelistCharacters - non alphanumeric characters that will not be removed
+     *      during normalization
+     */
+    public LenientComparisonStringNormalizer(List<Character> whitelistCharacters) {
+        this.whitelistCharacters = whitelistCharacters;
+    }
+    
+    
+    //------------------------ LOGIC --------------------------
+
+    
+    
+    /**
+     * Normalizes the given value. <br/>
+     * The normalized strings are better suited for non-strict comparisons, in which one does NOT care about characters that are
+     * neither letters nor digits; about accidental spaces or different diacritics etc. <br/><br/>
+     * This method:
+     * <ul>
+     * <li>Replaces all characters that are not letters or digits with spaces (except those on whitelist characters list)</li>
+     * <li>Replaces white spaces with spaces </li>
+     * <li>Trims</li>
+     * <li>Compacts multi-space gaps to one-space gaps</li>
+     * <li>Removes diacritics</li>
+     * <li>Changes characters to lower case</li>
+     * </ul>
+     * Returns "" if the passed value is null or blank
+     *
+     * @param value the string to normalize 
+     * @see DiacriticsRemover#removeDiacritics(String, boolean)
+     *
+     *
+     */
+    public String normalize(String value) {
+        
+        if (StringUtils.isBlank(value)) {
+        
+            return "";
+
+        }
+        
+        
+        String result = value;
+        
+        result = DiacriticsRemover.removeDiacritics(result);
+        
+        result = removeNonLetterDigitCharacters(result);
+        
+        result = result.toLowerCase();
+        
+        result = result.trim().replaceAll(" +", " ");
+        
+        return result;
+    }
+    
+    
+    
+    
+    //------------------------ PRIVATE --------------------------
+
+    
+    private String removeNonLetterDigitCharacters(final String value) {
+        
+        StringBuilder sb = new StringBuilder();
+        
+        for (int i = 0; i < value.length(); ++i) {
+   
+            char c = value.charAt(i);
+            
+            if (Character.isLetterOrDigit(c) || whitelistCharacters.contains(c)) {
+                sb.append(c);
+            } else {
+                sb.append(' ');
+            }
+        }
+        
+        return sb.toString();
+    }
+
+ 
+
+}
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MDStoreRecordsImporter.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MDStoreRecordsImporter.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MDStoreRecordsImporter.java	(revision 51263)
@@ -0,0 +1,157 @@
+package eu.dnetlib.dhp.wf.importer.mdrecord;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+
+import com.google.common.base.Preconditions;
+import eu.dnetlib.dhp.common.WorkflowRuntimeParameters;
+import eu.dnetlib.dhp.common.counter.NamedCounters;
+import eu.dnetlib.dhp.common.counter.NamedCountersFileWriter;
+import eu.dnetlib.dhp.common.java.PortBindings;
+import eu.dnetlib.dhp.common.java.Process;
+import eu.dnetlib.dhp.common.java.io.DataStore;
+import eu.dnetlib.dhp.common.java.io.FileSystemPath;
+import eu.dnetlib.dhp.common.java.porttype.AvroPortType;
+import eu.dnetlib.dhp.common.java.porttype.PortType;
+import eu.dnetlib.dhp.importer.schemas.ImportedRecord;
+import eu.dnetlib.dhp.importer.schemas.RecordFormat;
+import eu.dnetlib.dhp.wf.importer.facade.MDStoreFacade;
+import eu.dnetlib.dhp.wf.importer.facade.ServiceFacadeUtils;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.log4j.Logger;
+import org.xml.sax.InputSource;
+
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.OOZIE_ACTION_OUTPUT_FILENAME;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_MDSTORE_IDS_CSV;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_MDSTORE_RECORD_MAXLENGTH;
+
+/**
+ * {@link MDStoreFacade} based metadata records importer.
+ * @author mhorst
+ *
+ */
+public class MDStoreRecordsImporter implements Process {
+
+    protected static final String COUNTER_NAME_TOTAL = "TOTAL";
+    
+    protected static final String COUNTER_NAME_SIZE_EXCEEDED = "SIZE_EXCEEDED";
+    
+    protected static final String PORT_OUT_MDRECORDS = "mdrecords";
+    
+    private static final Logger log = Logger.getLogger(MDStoreRecordsImporter.class);
+    
+    private final static int progressLogInterval = 100000;
+    
+    private final NamedCountersFileWriter countersWriter = new NamedCountersFileWriter();
+    
+    private final Map<String, PortType> outputPorts = new HashMap<String, PortType>();
+
+    
+    //------------------------ CONSTRUCTORS -------------------
+    
+    public MDStoreRecordsImporter() {
+        outputPorts.put(PORT_OUT_MDRECORDS, new AvroPortType(ImportedRecord.SCHEMA$));
+    }
+    
+    //------------------------ LOGIC --------------------------
+    
+    @Override
+    public Map<String, PortType> getInputPorts() {
+        return Collections.emptyMap();
+    }
+
+    @Override
+    public Map<String, PortType> getOutputPorts() {
+        return outputPorts;
+    }
+
+    @Override
+    public void run(PortBindings portBindings, Configuration conf,
+            Map<String, String> parameters) throws Exception {
+        
+        Preconditions.checkArgument(parameters.containsKey(IMPORT_MDSTORE_IDS_CSV), 
+                "unknown mdstore identifier, required parameter '%s' is missing!", IMPORT_MDSTORE_IDS_CSV);
+        String mdStoreIdsCSV = parameters.get(IMPORT_MDSTORE_IDS_CSV);
+        int recordMaxLength = parameters.containsKey(IMPORT_MDSTORE_RECORD_MAXLENGTH)?
+                Integer.parseInt(parameters.get(IMPORT_MDSTORE_RECORD_MAXLENGTH)):Integer.MAX_VALUE;
+        
+        NamedCounters counters = new NamedCounters(new String[] { COUNTER_NAME_TOTAL, COUNTER_NAME_SIZE_EXCEEDED });
+        
+        if (StringUtils.isNotBlank(mdStoreIdsCSV) && !WorkflowRuntimeParameters.UNDEFINED_NONEMPTY_VALUE.equals(mdStoreIdsCSV)) {
+            
+            String[] mdStoreIds = StringUtils.split(mdStoreIdsCSV, WorkflowRuntimeParameters.DEFAULT_CSV_DELIMITER);
+
+            try (DataFileWriter<ImportedRecord> recordWriter = getWriter(FileSystem.get(conf), portBindings)) {
+                
+                MDStoreFacade mdStoreFacade = ServiceFacadeUtils.instantiate(parameters);
+                
+                SAXParserFactory parserFactory = SAXParserFactory.newInstance();
+                parserFactory.setNamespaceAware(true);
+                SAXParser saxParser = parserFactory.newSAXParser();
+                MDRecordHandler mdRecordHandler = new MDRecordHandler();
+
+                long startTime = System.currentTimeMillis();
+                int currentCount = 0;
+                
+                for (String mdStoreId : mdStoreIds) {
+                    for (String mdRecord : mdStoreFacade.deliverMDRecords(mdStoreId)) {
+                        if (!StringUtils.isEmpty(mdRecord)) {
+                            if (mdRecord.length() <= recordMaxLength) {
+                                saxParser.parse(new InputSource(new StringReader(mdRecord)), mdRecordHandler);
+                                String recordId = mdRecordHandler.getRecordId();
+                                if (StringUtils.isNotBlank(recordId)) {
+                                    recordWriter.append(
+                                            ImportedRecord.newBuilder()
+                                                .setId(recordId)
+                                                .setBody(mdRecord)
+                                                .setFormat(RecordFormat.XML)
+                                                .build());
+                                    counters.increment(COUNTER_NAME_TOTAL);
+                                } else {
+                                    log.error("skipping, unable to extract identifier from record: " + mdRecord);
+                                }    
+                            } else {
+                                counters.increment(COUNTER_NAME_SIZE_EXCEEDED);
+                                log.error("mdstore record maximum length (" + recordMaxLength + "): was exceeded: "
+                                        + mdRecord.length() + ", record content:\n" + mdRecord);
+                            }
+                            
+                        } else {
+                            log.error("got empty metadata record from mdstore: " + mdStoreId);
+                        }
+                        currentCount++;
+                        if (currentCount % progressLogInterval == 0) {
+                            log.info("current progress: " + currentCount + ", last package of " + progressLogInterval
+                                    + " processed in " + ((System.currentTimeMillis() - startTime) / 1000) + " secs");
+                            startTime = System.currentTimeMillis();
+                        }
+                    } 
+                }
+                log.info("total number of processed records: " + currentCount);
+            }
+        }
+        
+        if (counters.currentValue(COUNTER_NAME_TOTAL)==0) {
+            log.warn("parsed 0 metadata records from mdstores: " + mdStoreIdsCSV);
+        }
+        countersWriter.writeCounters(counters, System.getProperty(OOZIE_ACTION_OUTPUT_FILENAME));
+
+    }
+    
+    /**
+     * Provides {@link ImportedRecord} writer consuming records.
+     */
+    protected DataFileWriter<ImportedRecord> getWriter(FileSystem fs, PortBindings portBindings) throws IOException {
+        return DataStore.create(
+                new FileSystemPath(fs, portBindings.getOutput().get(PORT_OUT_MDRECORDS)), ImportedRecord.SCHEMA$);
+    }
+
+}
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/MDStoreFacade.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/MDStoreFacade.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/MDStoreFacade.java	(revision 51263)
@@ -0,0 +1,17 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+/**
+ * MDStore service facade.
+ * 
+ * @author mhorst
+ *
+ */
+public interface MDStoreFacade {
+
+    /**
+     * Delivers all records for given MDStore identifier
+     * @param mdStoreId MDStore identifier
+     */
+    Iterable<String> deliverMDRecords(String mdStoreId) throws ServiceFacadeException;
+    
+}
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacade.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacade.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacade.java	(revision 51263)
@@ -0,0 +1,52 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreService;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
+import eu.dnetlib.enabling.tools.JaxwsServiceResolverImpl;
+
+/**
+ * WebService based ObjectStore facade.
+ * 
+ * @author mhorst
+ *
+ */
+public class WebServiceObjectStoreFacade extends AbstractResultSetAwareWebServiceFacade<ObjectStoreService> implements ObjectStoreFacade {
+
+    
+    //------------------------ CONSTRUCTORS -------------------
+    
+    /**
+     * @param serviceLocation ObjectStore webservice location
+     * @param serviceReadTimeout service read timeout
+     * @param serviceConnectionTimeout service connection timeout
+     * @param resultSetReadTimeout resultset read timeout
+     * @param resultSetConnectionTimeout result set connection timeout
+     * @param resultSetPageSize resultset page size
+     */
+    public WebServiceObjectStoreFacade(String serviceLocation, 
+            long serviceReadTimeout, long serviceConnectionTimeout,
+            long resultSetReadTimeout, long resultSetConnectionTimeout, int resultSetPageSize) {
+        super(ObjectStoreService.class, serviceLocation, 
+                serviceReadTimeout, serviceConnectionTimeout, 
+                resultSetReadTimeout, resultSetConnectionTimeout, resultSetPageSize);
+    }
+    
+    //------------------------ LOGIC --------------------------
+    
+    @Override
+    public Iterable<String> deliverObjects(String objectStoreId, long from, long until) throws ServiceFacadeException {
+        try {
+            W3CEndpointReference eprResult = getService().deliverObjects(objectStoreId, from, until);
+            ResultSetClientFactory rsFactory = new ResultSetClientFactory(
+                    getResultSetPageSize(), getResultSetReadTimeout(), getResultSetConnectionTimeout());
+            rsFactory.setServiceResolver(new JaxwsServiceResolverImpl());
+            return rsFactory.getClient(eprResult);
+        } catch (ObjectStoreServiceException e) {
+            throw new ServiceFacadeException("delivering records for object store " + objectStoreId + " failed!", e);
+        }
+    }
+
+}
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ISLookupFacade.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ISLookupFacade.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ISLookupFacade.java	(revision 51263)
@@ -0,0 +1,17 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+/**
+ * ISLookup service facade.
+ * 
+ * @author mhorst
+ *
+ */
+public interface ISLookupFacade {
+
+    /**
+     * Provides all profiles matching given query
+     * @param xPathQuery XPath query
+     */
+    Iterable<String> searchProfile(String xPathQuery) throws ServiceFacadeException;
+    
+}
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiverWithCounter.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiverWithCounter.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiverWithCounter.java	(revision 51263)
@@ -0,0 +1,50 @@
+package eu.dnetlib.dhp.wf.importer;
+
+import java.io.IOException;
+
+import org.apache.avro.file.DataFileWriter;
+
+/**
+ * {@link DataFileWriter} based record receiver with counter of
+ * received records.
+ * 
+ * @author madryk
+ */
+public class DataFileRecordReceiverWithCounter<T> extends DataFileRecordReceiver<T> {
+
+    private long receivedCount = 0L;
+    
+    
+    //------------------------ CONSTRUCTORS --------------------------
+    
+    /**
+     * Default constructor
+     * 
+     * @param writer - writer of the received records
+     */
+    public DataFileRecordReceiverWithCounter(DataFileWriter<T> writer) {
+        super(writer);
+    }
+    
+    
+    //------------------------ GETTERS --------------------------
+    
+    /**
+     * Returns number of received records
+     */
+    public long getReceivedCount() {
+        return receivedCount;
+    }
+    
+    
+    //------------------------ LOGIC --------------------------
+    
+    /**
+     * Receives passed record and increments the counter.
+     */
+    @Override
+    public void receive(T record) throws IOException {
+        super.receive(record);
+        ++receivedCount;
+    }
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/DiacriticsRemover.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/DiacriticsRemover.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/DiacriticsRemover.java	(revision 51263)
@@ -0,0 +1,113 @@
+/*
+ * This file is part of CoAnSys project.
+ * Copyright (c) 2012-2015 ICM-UW
+ * 
+ * CoAnSys is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+
+ * CoAnSys is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with CoAnSys. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package eu.dnetlib.dhp.common.string;
+
+import java.text.Normalizer;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Mapping to the basic Latin alphabet (a-z, A-Z). In most cases, a character is
+ * mapped to the closest visual form, rather than functional one, e.g.: "ö" is
+ * mapped to "o" rather than "oe", and "đ" is mapped to "d" rather than "dj" or
+ * "gj". Notable exceptions include: "ĸ" mapped to "q", "ß" mapped to "ss", and
+ * "Þ", "þ" mapped to "Y", "y".
+ *
+ * <p> Each character is processed as follows: <ol> <li>the character is
+ * compatibility decomposed,</li> <li>all the combining marks are removed,</li>
+ * <li>the character is compatibility composed,</li> <li>additional "manual"
+ * substitutions are applied.</li> </ol> </p>
+ *
+ * <p> All the characters from the "Latin-1 Supplement" and "Latin Extended-A"
+ * Unicode blocks are mapped to the "Basic Latin" block. Characters from other
+ * alphabets are generally left intact, although the decomposable ones may be
+ * affected by the procedure. </p>
+ *
+ * @author Lukasz Bolikowski (bolo@icm.edu.pl)
+ * 
+ * @author Łukasz Dumiszewski /just copied from coansys-commons/
+ *
+ */
+public final class DiacriticsRemover {
+
+    private static final Character[] from = {
+        'Æ', 'Ð', 'Ø', 'Þ', 'ß', 'æ', 'ð', 'ø', 'þ', 'Đ', 'đ', 'Ħ',
+        'ħ', 'ı', 'ĸ', 'Ł', 'ł', 'Ŋ', 'ŋ', 'Œ', 'œ', 'Ŧ', 'ŧ'};
+    private static final String[] to = {
+        "AE", "D", "O", "Y", "ss", "ae", "d", "o", "y", "D", "d", "H",
+        "h", "i", "q", "L", "l", "N", "n", "OE", "oe", "T", "t"};
+    
+    private static Map<Character, String> lookup = buildLookup();
+    
+    
+    //------------------------ CONSTRUCTORS -------------------
+    
+    
+    private DiacriticsRemover() {}
+    
+
+    //------------------------ LOGIC --------------------------
+    
+    
+    /**
+     * Removes diacritics from a text.
+     *
+     * @param text Text to process.
+     * @return Text without diacritics.
+     */
+    public static String removeDiacritics(String text) {
+        if (text == null) {
+            return null;
+        }
+
+        String tmp = Normalizer.normalize(text, Normalizer.Form.NFKD);
+
+        StringBuilder builder = new StringBuilder();
+        for (int i = 0; i < tmp.length(); i++) {
+            Character ch = tmp.charAt(i);
+            if (Character.getType(ch) == Character.NON_SPACING_MARK) {
+                continue;
+            }
+
+            if (lookup.containsKey(ch)) {
+                builder.append(lookup.get(ch));
+            } else {
+                builder.append(ch);
+            }
+        }
+
+        return builder.toString();
+    }
+
+
+    //------------------------ PRIVATE --------------------------
+    
+    private static Map<Character, String> buildLookup() {
+        if (from.length != to.length) {
+            throw new IllegalStateException();
+        }
+
+        Map<Character, String> _lookup = new HashMap<Character, String>();
+        for (int i = 0; i < from.length; i++) {
+            _lookup.put(from[i], to[i]);
+        }
+
+        return _lookup;
+    }
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/report/ReportEntryFactory.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/report/ReportEntryFactory.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/report/ReportEntryFactory.java	(revision 51263)
@@ -0,0 +1,32 @@
+package eu.dnetlib.dhp.common.report;
+
+import eu.dnetlib.dhp.common.schemas.ReportEntry;
+import eu.dnetlib.dhp.common.schemas.ReportEntryType;
+
+/**
+ * Factory of {@link ReportEntry} objects.
+ * 
+ * @author madryk
+ */
+public final class ReportEntryFactory {
+
+    // ----------------------- CONSTRUCTORS -----------------------------
+    
+    private ReportEntryFactory() {}
+
+    // ----------------------- LOGIC ------------------------------------
+    
+    /**
+     * Creates {@link ReportEntry} with {@link ReportEntryType#COUNTER} type
+     */
+    public static ReportEntry createCounterReportEntry(String key, long count) {
+        return new ReportEntry(key, ReportEntryType.COUNTER, String.valueOf(count));
+    }
+    
+    /**
+     * Creates {@link ReportEntry} with {@link ReportEntryType#DURATION} type
+     */
+    public static ReportEntry createDurationReportEntry(String key, long duration) {
+        return new ReportEntry(key, ReportEntryType.DURATION, String.valueOf(duration));
+    }
+}
Index: dnet-hadoop/dhp-wf/dhp-wf-import/pom.xml
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/pom.xml	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/pom.xml	(revision 51263)
@@ -0,0 +1,105 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<parent>
+		<groupId>eu.dnetlib.dhp</groupId>
+		<artifactId>dhp-wf</artifactId>
+		<version>1.0.0-SNAPSHOT</version>
+	</parent>
+	<modelVersion>4.0.0</modelVersion>
+
+	<artifactId>dhp-wf-import</artifactId>
+
+	<dependencies>
+
+		<dependency>
+			<groupId>${project.groupId}</groupId>
+			<artifactId>dhp-common</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+
+		<dependency>
+			<groupId>${project.groupId}</groupId>
+			<artifactId>dhp-common</artifactId>
+			<version>${project.version}</version>
+			<type>test-jar</type>
+			<scope>test</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>${project.groupId}</groupId>
+			<artifactId>dhp-schemas</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+
+		<!-- required after introducing 'provided' scope for hadoop libs -->
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-mapreduce-client-core</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-common</artifactId>
+		</dependency>
+
+		<dependency>
+			<groupId>com.googlecode.json-simple</groupId>
+			<artifactId>json-simple</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>commons-cli</groupId>
+			<artifactId>commons-cli</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>eu.dnetlib</groupId>
+			<artifactId>dnet-objectstore-rmi</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>eu.dnetlib</groupId>
+			<artifactId>cnr-rmi-api</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>eu.dnetlib</groupId>
+			<artifactId>cnr-resultset-client</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>eu.dnetlib</groupId>
+			<artifactId>dnet-openaireplus-mapping-utils</artifactId>
+		</dependency>
+		<!-- proper spring context version required by cnr-resultset-client -->
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-context</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.cxf</groupId>
+			<artifactId>cxf-rt-frontend-jaxws</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>com.google.code.gson</groupId>
+			<artifactId>gson</artifactId>
+		</dependency>
+
+		<dependency>
+			<groupId>org.apache.spark</groupId>
+			<artifactId>spark-core_2.10</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.spark</groupId>
+			<artifactId>spark-sql_2.10</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>com.databricks</groupId>
+			<artifactId>spark-avro_2.10</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.mongodb.spark</groupId>
+			<artifactId>mongo-spark-connector_2.10</artifactId>
+		</dependency>
+
+
+	</dependencies>
+
+
+</project>
Index: dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/src/test/resources/eu/dnetlib/maven/plugin/properties/included.properties
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/src/test/resources/eu/dnetlib/maven/plugin/properties/included.properties	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/src/test/resources/eu/dnetlib/maven/plugin/properties/included.properties	(revision 51263)
@@ -0,0 +1 @@
+includedPropertyKey=irrelevantValue
\ No newline at end of file
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/FileSystemPath.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/FileSystemPath.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/FileSystemPath.java	(revision 51263)
@@ -0,0 +1,61 @@
+package eu.dnetlib.dhp.common.java.io;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Path to a directory or a file along with information about the 
+ * file system in which the path is defined.
+ * 
+ * @author Mateusz Kobos
+ *
+ */
+public class FileSystemPath {
+	private final FileSystem fs;
+	private final Path path;
+
+	/**
+	 * Path in the local file system
+	 */
+	public FileSystemPath(File file) throws IOException {
+		this(new Path(file.toURI()));
+	}
+	
+	/**
+	 * Path in the local file system
+	 */
+	public FileSystemPath(Path path) throws IOException{
+		this(FileSystem.get(new Configuration(false)), path);
+	}
+	
+	/**
+	 * Path in the given file system
+	 */
+	public FileSystemPath(FileSystem fs, Path path){
+		this.fs = fs;
+		this.path = path;
+	}
+	
+	/** Create a path with a child element */
+	public FileSystemPath(FileSystemPath parent, String child){
+		this.fs = parent.getFileSystem();
+		this.path = new Path(parent.getPath(), child);
+	}
+
+	public FileSystem getFileSystem() {
+		return fs;
+	}
+
+	public Path getPath() {
+		return path;
+	}
+	
+	public FSDataInputStream getInputStream() throws IOException{
+		return getFileSystem().open(getPath());
+	}
+}
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/resources/eu/dnetlib/dhp/wf/importer/mdrecord/oozie_app/workflow.xml
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/resources/eu/dnetlib/dhp/wf/importer/mdrecord/oozie_app/workflow.xml	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/resources/eu/dnetlib/dhp/wf/importer/mdrecord/oozie_app/workflow.xml	(revision 51263)
@@ -0,0 +1,124 @@
+<workflow-app xmlns="uri:oozie:workflow:0.4" name="importer_mdrecord">
+	
+	<parameters>
+        <property>
+            <name>mdstore_facade_factory_classname</name>
+            <value>eu.dnetlib.dhp.wf.importer.facade.WebServiceMDStoreFacadeFactory</value>
+            <description>ServiceFacadeFactory implementation class name producing eu.dnetlib.dhp.wf.importer.facade.WebServiceMDStoreFacade</description>
+        </property>
+		<property>
+			<name>mdstore_service_location</name>
+            <value>$UNDEFINED$</value>
+			<description>mdstore service location</description>
+		</property>
+		<property>
+			<name>mdstore_ids_csv</name>
+            <value>$UNDEFINED$</value>
+			<description>comma separated mdstore identifiers</description>
+		</property>
+        <property>
+            <name>mdstore_record_maxlength</name>
+            <value>500000</value>
+            <description>maximum allowed length of mdstore record</description>
+        </property>
+        <property>
+            <name>output</name>
+            <description>ImportedRecord avro datastore output holding mdrecords</description>
+        </property>
+        <property>
+            <name>output_report_root_path</name>
+            <description>base directory for storing reports</description>
+        </property>
+        <property>
+            <name>output_report_relative_path</name>
+            <value>import_mdrecord</value>
+            <description>directory for storing report (relative to output_report_root_path)</description>
+        </property>
+        <property>
+            <name>dnet_service_client_read_timeout</name>
+            <value>60000</value>
+            <description>DNet service client reading timeout (expressed in milliseconds)</description>
+        </property>
+        <property>
+            <name>dnet_service_client_connection_timeout</name>
+            <value>60000</value>
+            <description>DNet service client connection timeout (expressed in milliseconds)</description>
+        </property>
+        <property>
+            <name>resultset_client_read_timeout</name>
+            <value>60000</value>
+            <description>result set client reading timeout (expressed in milliseconds)</description>
+        </property>
+        <property>
+            <name>resultset_client_connection_timeout</name>
+            <value>60000</value>
+            <description>result set client connection timeout (expressed in milliseconds)</description>
+        </property>
+        <property>
+            <name>report_properties_prefix</name>
+            <value>import.mdrecord</value>
+            <description>report entry related to total number of imported records</description>
+        </property>
+	</parameters>
+
+    <global>
+        <job-tracker>${jobTracker}</job-tracker>
+        <name-node>${nameNode}</name-node>
+        <configuration>
+            <property>
+                <name>mapreduce.job.queuename</name>
+                <value>${queueName}</value>
+            </property>
+            <property>
+                <name>oozie.launcher.mapred.job.queue.name</name>
+                <value>${oozieLauncherQueueName}</value>
+            </property>
+        </configuration>
+    </global>
+    
+	<start to="mdrecord-importer" />
+	
+	<action name="mdrecord-importer">
+		<java>
+			<!-- The data generated by this node is deleted in this section -->
+			<prepare>
+				<delete path="${nameNode}${output}" />
+			</prepare>
+			<main-class>eu.dnetlib.dhp.common.java.ProcessWrapper</main-class>
+			<arg>eu.dnetlib.dhp.wf.importer.mdrecord.MDStoreRecordsImporter</arg>
+            
+			<arg>-Pimport.mdstore.service.location=${mdstore_service_location}</arg>
+			<arg>-Pimport.mdstore.ids.csv=${mdstore_ids_csv}</arg>
+            <arg>-Pimport.mdstore.record.maxlength=${mdstore_record_maxlength}</arg>
+
+            <arg>-Pimport.resultset.client.read.timeout=${resultset_client_read_timeout}</arg>
+            <arg>-Pimport.resultset.client.connection.timeout=${resultset_client_connection_timeout}</arg>
+            <arg>-Pdnet.service.client.read.timeout=${dnet_service_client_read_timeout}</arg>
+            <arg>-Pdnet.service.client.connection.timeout=${dnet_service_client_connection_timeout}</arg>
+            
+            <arg>-Pimport.facade.factory.classname=${mdstore_facade_factory_classname}</arg>
+			<arg>-Omdrecords=${output}</arg>
+			<capture-output />
+		</java>
+		<ok to="report" />
+		<error to="fail" />
+	</action>
+    
+    <action name="report">
+        <java>
+            <main-class>eu.dnetlib.dhp.common.java.ProcessWrapper</main-class>
+            <arg>eu.dnetlib.dhp.common.report.ReportGenerator</arg>
+            <arg>-Preport.${report_properties_prefix}.total=${wf:actionData('mdrecord-importer')['TOTAL']}</arg>
+            <arg>-Preport.${report_properties_prefix}.invalid.sizeExceeded=${wf:actionData('mdrecord-importer')['SIZE_EXCEEDED']}</arg>
+            <arg>-Oreport=${output_report_root_path}/${output_report_relative_path}</arg>
+        </java>
+        <ok to="end" />
+        <error to="fail" />
+    </action>
+    
+	<kill name="fail">
+		<message>Unfortunately, the process failed -- error message:
+			[${wf:errorMessage(wf:lastErrorNode())}]</message>
+	</kill>
+	<end name="end" />
+</workflow-app>
Index: dnet-hadoop/dhp-wf/pom.xml
===================================================================
--- dnet-hadoop/dhp-wf/pom.xml	(nonexistent)
+++ dnet-hadoop/dhp-wf/pom.xml	(revision 51263)
@@ -0,0 +1,249 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+
+	<parent>
+		<groupId>eu.dnetlib.dhp</groupId>
+		<artifactId>dhp</artifactId>
+		<version>1.0.0-SNAPSHOT</version>
+	</parent>
+
+	<artifactId>dhp-wf</artifactId>
+	<packaging>pom</packaging>
+
+	<modules>
+		<module>dhp-wf-import</module>
+	</modules>
+
+	<properties>
+		<maven.build.timestamp.format>yyyy-MM-dd_HH_mm</maven.build.timestamp.format>
+		<!-- default Oozie installer properties required to be defined at pom.xml level -->
+		<!-- other project properties are defined in project-default.properties -->
+		<oozie.package.file.name>oozie-package</oozie.package.file.name>
+		<!-- notice: sandboxName is generated based on workflow.source.dir property -->
+		<workflow.source.dir>src/test/resources/define/path/pointing/to/directory/holding/oozie_app</workflow.source.dir>
+		<oozieAppDir>oozie_app</oozieAppDir>
+		<queueName>default</queueName>
+		<importerQueueName>default</importerQueueName>
+		<oozieLauncherQueueName>default</oozieLauncherQueueName>
+		<primed.dir>primed</primed.dir>
+
+		<oozie.package.dependencies.include.scope>runtime</oozie.package.dependencies.include.scope>
+		<oozie.package.dependencies.exclude.scope></oozie.package.dependencies.exclude.scope>
+		<oozie.package.skip.test.jar>true</oozie.package.skip.test.jar>
+
+		<dhpConnectionProperties>${user.home}/.dhp/application.properties</dhpConnectionProperties>
+
+		<output.dir.name>${maven.build.timestamp}</output.dir.name>
+
+		<projectVersion>${project.version}</projectVersion>
+		<oozie.use.system.libpath>true</oozie.use.system.libpath>
+	</properties>
+
+	<dependencies>
+
+		<!--
+		<dependency>
+			<groupId>eu.dnetlib.iis</groupId>
+			<artifactId>iis-build-assembly-resources</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+		-->
+		<dependency>
+			<groupId>org.apache.oozie</groupId>
+			<artifactId>oozie-client</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>net.schmizz</groupId>
+			<artifactId>sshj</artifactId>
+			<scope>test</scope>
+		</dependency>
+	</dependencies>
+
+	<profiles>
+		<profile>
+			<id>oozie-package</id>
+			<build>
+				<plugins>
+					<plugin>
+						<groupId>org.apache.maven.plugins</groupId>
+						<artifactId>maven-enforcer-plugin</artifactId>
+						<version>1.4.1</version>
+						<executions>
+							<execution>
+								<id>enforce-connection-properties-file-existence</id>
+								<phase>initialize</phase>
+								<goals>
+									<goal>enforce</goal>
+								</goals>
+								<configuration>
+									<rules>
+										<requireFilesExist>
+											<files>
+												<file>${dhpConnectionProperties}</file>
+											</files>
+											<message>
+												The file with connection properties could not be found. Please, create the ${dhpConnectionProperties} file or set the location to another already created file by using
+												-DdhpConnectionProperties property.
+											</message>
+										</requireFilesExist>
+									</rules>
+									<fail>true</fail>
+								</configuration>
+							</execution>
+						</executions>
+					</plugin>
+					<plugin>
+						<groupId>org.apache.maven.plugins</groupId>
+						<artifactId>maven-dependency-plugin</artifactId>
+						<executions>
+							<execution>
+								<id>copy dependencies</id>
+								<phase>prepare-package</phase>
+								<goals>
+									<goal>copy-dependencies</goal>
+								</goals>
+								<configuration>
+									<includeScope>${oozie.package.dependencies.include.scope}</includeScope>
+									<excludeScope>${oozie.package.dependencies.exclude.scope}</excludeScope>
+									<silent>true</silent>
+								</configuration>
+							</execution>
+						</executions>
+					</plugin>
+					<!-- Plugin originally defined in attach-test-resources It was moved here to ensure that it will execute before priming -->
+					<plugin>
+						<groupId>org.apache.maven.plugins</groupId>
+						<artifactId>maven-jar-plugin</artifactId>
+						<executions>
+							<execution>
+								<id>attach-test-resources-package</id>
+								<phase>prepare-package</phase>
+								<goals>
+									<goal>test-jar</goal>
+								</goals>
+								<configuration>
+									<skip>${oozie.package.skip.test.jar}</skip>
+								</configuration>
+							</execution>
+						</executions>
+					</plugin>
+
+
+					<plugin>
+						<groupId>pl.project13.maven</groupId>
+						<artifactId>git-commit-id-plugin</artifactId>
+						<version>2.1.11</version>
+						<executions>
+							<execution>
+								<goals>
+									<goal>revision</goal>
+								</goals>
+							</execution>
+						</executions>
+						<configuration>
+							<verbose>true</verbose>
+							<dateFormat>yyyy-MM-dd'T'HH:mm:ssZ</dateFormat>
+							<generateGitPropertiesFile>true</generateGitPropertiesFile>
+							<generateGitPropertiesFilename>target/${oozie.package.file.name}/${oozieAppDir}/version.properties</generateGitPropertiesFilename>
+						</configuration>
+					</plugin>
+					<plugin>
+						<groupId>org.apache.maven.plugins</groupId>
+						<artifactId>maven-assembly-plugin</artifactId>
+						<version>3.0.0</version>
+						<executions>
+							<execution>
+								<id>assembly-oozie-installer</id>
+								<phase>package</phase>
+								<goals>
+									<goal>single</goal>
+								</goals>
+								<configuration>
+									<appendAssemblyId>false</appendAssemblyId>
+									<finalName>${oozie.package.file.name}_shell_scripts</finalName>
+									<descriptorRefs>
+										<descriptorRef>oozie-installer</descriptorRef>
+									</descriptorRefs>
+								</configuration>
+							</execution>
+						</executions>
+					</plugin>
+
+					<plugin>
+						<!-- this plugin prepares oozie installer package-->
+
+						<artifactId>maven-antrun-plugin</artifactId>
+						<executions>
+							<!-- extracting shared resources phase -->
+							<execution>
+								<id>installer-copy-custom</id>
+								<phase>process-resources</phase>
+								<goals>
+									<goal>run</goal>
+								</goals>
+								<configuration>
+									<tasks>
+										<property name="assembly-resources.loc" value="${maven.dependency.eu.dnetlib.iis.iis-build-assembly-resources.jar.path}" />
+										<unjar src="${assembly-resources.loc}" dest="${project.build.directory}/assembly-resources" />
+									</tasks>
+								</configuration>
+							</execution>
+							<!-- packaging phase -->
+							<execution>
+								<phase>package</phase>
+								<configuration>
+									<tasks>
+										<!-- copying workflow resources -->
+										<mkdir dir="target/${oozie.package.file.name}" />
+										<mkdir dir="target/${oozie.package.file.name}/${oozieAppDir}" />
+										<copy todir="target/${oozie.package.file.name}/${oozieAppDir}">
+											<!-- <fileset dir="${workflow.source.dir}/${oozieAppDir}" /> replacing with primed dir location -->
+											<fileset dir="target/${primed.dir}/${oozieAppDir}" />
+										</copy>
+										<!-- copying all jars to oozie lib directory -->
+										<mkdir dir="target/${oozie.package.file.name}/${oozieAppDir}/lib" />
+										<copy todir="target/${oozie.package.file.name}/${oozieAppDir}/lib">
+											<fileset dir="${project.build.directory}/dependency" />
+										</copy>
+										<!-- copying current module lib -->
+										<copy todir="target/${oozie.package.file.name}/${oozieAppDir}/lib">
+											<fileset dir="${project.build.directory}">
+												<include name="*.jar" />
+											</fileset>
+										</copy>
+
+
+										<fixcrlf srcdir="target/${oozie.package.file.name}/${oozieAppDir}/" encoding="UTF-8" outputencoding="UTF-8" includes="**/*.sh,**/*.json,**/*.py,**/*.sql" eol="lf"/>
+
+
+										<!-- creating tar.gz package -->
+										<tar destfile="target/${oozie.package.file.name}.tar.gz" compression="gzip">
+											<tarfileset dir="target/${oozie.package.file.name}" />
+											<tarfileset dir="target/${oozie.package.file.name}_shell_scripts" filemode="0755">
+												<include name="**/*.sh" />
+											</tarfileset>
+											<tarfileset dir="target/${oozie.package.file.name}_shell_scripts" filemode="0644">
+												<exclude name="**/*.sh" />
+											</tarfileset>
+										</tar>
+										<!-- cleanup -->
+										<delete dir="target/${oozie.package.file.name}" />
+										<delete dir="target/${oozie.package.file.name}_shell_scripts" />
+									</tasks>
+								</configuration>
+								<goals>
+									<goal>run</goal>
+								</goals>
+							</execution>
+						</executions>
+					</plugin>
+				</plugins>
+			</build>
+		</profile>
+
+	</profiles>
+
+</project>
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/AvroDataStoreReader.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/AvroDataStoreReader.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/AvroDataStoreReader.java	(revision 51263)
@@ -0,0 +1,156 @@
+package eu.dnetlib.dhp.common.java.io;
+
+import java.io.IOException;
+import java.util.NoSuchElementException;
+import java.util.regex.Pattern;
+
+import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileReader;
+import org.apache.avro.specific.SpecificDatumReader;
+import org.apache.hadoop.fs.AvroFSInput;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.RemoteIterator;
+
+
+
+/**
+ * An abstraction over data store format which allows
+ * iterating over records stored in the data store. 
+ * It handles the standard case of a data store that is a directory containing 
+ * many Avro files (but it can also read records from a single file). 
+ * 
+ * @author mhorst
+ * @author Mateusz Kobos
+ */
+class AvroDataStoreReader<T> implements CloseableIterator<T> {
+
+	private DataFileReader<T> currentReader;
+	private RemoteIterator<LocatedFileStatus> fileIterator;
+	private final FileSystemPath path;
+	private final Schema readerSchema;
+
+	/**
+	 * Ignore file starting with underscore. Such files are also ignored by
+	 * default by map-reduce jobs.
+	 */
+	private final Pattern whitelistPattern = Pattern.compile("^(?!_).*");
+
+	/**
+	 * Here the schema used for reading the data store is set to be the same 
+	 * as the one that was used to write it.
+	 */
+	public AvroDataStoreReader(final FileSystemPath path)
+			throws IOException {
+		this(path, null);
+	}
+	
+	/**
+	 * @param path path to the data store to be read
+	 * @param readerSchema the schema onto which the read data store will 
+	 * 	be projected
+	 */
+	public AvroDataStoreReader(final FileSystemPath path, Schema readerSchema)
+			throws IOException {
+		this.path = path;
+		this.readerSchema = readerSchema;
+		fileIterator = path.getFileSystem().listFiles(path.getPath(), false);
+		currentReader = getNextNonemptyReader();
+	}
+	
+	private DataFileReader<T> getNextNonemptyReader() throws IOException {
+		while (fileIterator != null && fileIterator.hasNext()) {
+			LocatedFileStatus currentFileStatus = fileIterator.next();
+			if (isValidFile(currentFileStatus)) {
+				FileSystemPath currPath = new FileSystemPath(
+						path.getFileSystem(), currentFileStatus.getPath());
+				DataFileReader<T> reader = 
+						getSingleFileReader(currPath, readerSchema);
+				/** Check if the file contains at least one record */
+				if(reader.hasNext()){
+					return reader;
+				} else {
+					reader.close();
+				}
+			}
+		}
+		/** fallback */
+		return null;
+	}
+	
+	/**
+	 * Get a reader for the specified Avro file. A utility function. 
+	 * @param path path to the existing file
+	 * @param readerSchema optional reader schema. If you want to use the
+	 *		default option of using writer schema as the reader schema, pass the
+	 *		{@code null} value. 
+	 * @throws IOException
+	 */
+	private static <T> DataFileReader<T> getSingleFileReader(
+			FileSystemPath path, Schema readerSchema) throws IOException{
+		try{
+		SpecificDatumReader<T> datumReader = new SpecificDatumReader<T>();
+		if(readerSchema != null){
+			datumReader.setExpected(readerSchema);
+		}
+		long len = path.getFileSystem().getFileStatus(path.getPath()).getLen();
+		FSDataInputStream inputStream = path.getFileSystem().open(path.getPath());
+		return new DataFileReader<T>(
+				new AvroFSInput(inputStream, len), datumReader);
+		} catch (IOException ex){
+			throw new IOException("Problem with file \""+
+					path.getPath().toString()+"\": "+ex.getMessage(), ex);
+		}
+	}
+
+	/**
+	 * Checks whether file is valid
+	 * 
+	 * @param fileStatus
+	 * @return true when valid, false otherwise
+	 */
+	private boolean isValidFile(LocatedFileStatus fileStatus) {
+		if (fileStatus.isFile()) {
+			return whitelistPattern.matcher(
+					fileStatus.getPath().getName()).matches();
+		}
+		/** fallback */
+		return false;
+	}
+
+	@Override
+	public boolean hasNext() {
+		return currentReader != null;
+	}
+
+	@Override
+	public T next(){
+		if(currentReader == null){
+			throw new NoSuchElementException();
+		}
+		T obj = currentReader.next();
+		if(!currentReader.hasNext()){
+			try{
+				currentReader.close();
+				currentReader = getNextNonemptyReader();
+			} catch(IOException ex){
+				throw new RuntimeException(ex);
+			}
+		}
+		return obj;
+	}
+
+	@Override
+	public void remove() {
+		throw new UnsupportedOperationException();
+	}
+
+	@Override
+	public void close() throws IOException {
+		if(currentReader != null){
+			currentReader.close();
+			currentReader = null;
+		}
+		fileIterator = null;
+	}
+}
\ No newline at end of file
Index: dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectPropertiesTest.java
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectPropertiesTest.java	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectPropertiesTest.java	(revision 51263)
@@ -0,0 +1,365 @@
+package eu.dnetlib.maven.plugin.properties;
+
+import static eu.dnetlib.maven.plugin.properties.WritePredefinedProjectProperties.PROPERTY_PREFIX_ENV;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.doReturn;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.project.MavenProject;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+
+/**
+ * @author mhorst
+ *
+ */
+@RunWith(MockitoJUnitRunner.class)
+public class WritePredefinedProjectPropertiesTest {
+
+    @Rule
+    public TemporaryFolder testFolder = new TemporaryFolder();
+    
+    @Mock
+    private MavenProject mavenProject;
+    
+    private WritePredefinedProjectProperties mojo;
+
+    @Before
+    public void init() {
+        mojo = new WritePredefinedProjectProperties();
+        mojo.outputFile = getPropertiesFileLocation();
+        mojo.project = mavenProject;
+        doReturn(new Properties()).when(mavenProject).getProperties();
+    }
+
+    // ----------------------------------- TESTS ---------------------------------------------
+    
+    @Test
+    public void testExecuteEmpty() throws Exception {
+        // execute
+        mojo.execute();
+        
+        // assert
+        assertTrue(mojo.outputFile.exists());
+        Properties storedProperties = getStoredProperties();
+        assertEquals(0, storedProperties.size());
+    }
+    
+    @Test
+    public void testExecuteWithProjectProperties() throws Exception {
+        // given
+        String key = "projectPropertyKey";
+        String value = "projectPropertyValue";
+        Properties projectProperties = new Properties();
+        projectProperties.setProperty(key, value);
+        doReturn(projectProperties).when(mavenProject).getProperties();
+        
+        // execute
+        mojo.execute();
+        
+        // assert
+        assertTrue(mojo.outputFile.exists());
+        Properties storedProperties = getStoredProperties();
+        assertEquals(1, storedProperties.size());
+        assertTrue(storedProperties.containsKey(key));
+        assertEquals(value, storedProperties.getProperty(key));
+    }
+    
+    @Test(expected=MojoExecutionException.class)
+    public void testExecuteWithProjectPropertiesAndInvalidOutputFile() throws Exception {
+        // given
+        String key = "projectPropertyKey";
+        String value = "projectPropertyValue";
+        Properties projectProperties = new Properties();
+        projectProperties.setProperty(key, value);
+        doReturn(projectProperties).when(mavenProject).getProperties();
+        mojo.outputFile = testFolder.getRoot();
+        
+        // execute
+        mojo.execute();
+    }
+    
+    @Test
+    public void testExecuteWithProjectPropertiesExclusion() throws Exception {
+        // given
+        String key = "projectPropertyKey";
+        String value = "projectPropertyValue";
+        String excludedKey = "excludedPropertyKey";
+        String excludedValue = "excludedPropertyValue";
+        Properties projectProperties = new Properties();
+        projectProperties.setProperty(key, value);
+        projectProperties.setProperty(excludedKey, excludedValue);
+        doReturn(projectProperties).when(mavenProject).getProperties();
+        mojo.setExclude(excludedKey);
+        
+        // execute
+        mojo.execute();
+        
+        // assert
+        assertTrue(mojo.outputFile.exists());
+        Properties storedProperties = getStoredProperties();
+        assertEquals(1, storedProperties.size());
+        assertTrue(storedProperties.containsKey(key));
+        assertEquals(value, storedProperties.getProperty(key));
+    }
+    
+    @Test
+    public void testExecuteWithProjectPropertiesInclusion() throws Exception {
+        // given
+        String key = "projectPropertyKey";
+        String value = "projectPropertyValue";
+        String includedKey = "includedPropertyKey";
+        String includedValue = "includedPropertyValue";
+        Properties projectProperties = new Properties();
+        projectProperties.setProperty(key, value);
+        projectProperties.setProperty(includedKey, includedValue);
+        doReturn(projectProperties).when(mavenProject).getProperties();
+        mojo.setInclude(includedKey);
+        
+        // execute
+        mojo.execute();
+        
+        // assert
+        assertTrue(mojo.outputFile.exists());
+        Properties storedProperties = getStoredProperties();
+        assertEquals(1, storedProperties.size());
+        assertTrue(storedProperties.containsKey(includedKey));
+        assertEquals(includedValue, storedProperties.getProperty(includedKey));
+    }
+    
+    @Test
+    public void testExecuteIncludingPropertyKeysFromFile() throws Exception {
+        // given
+        String key = "projectPropertyKey";
+        String value = "projectPropertyValue";
+        String includedKey = "includedPropertyKey";
+        String includedValue = "includedPropertyValue";
+        Properties projectProperties = new Properties();
+        projectProperties.setProperty(key, value);
+        projectProperties.setProperty(includedKey, includedValue);
+        doReturn(projectProperties).when(mavenProject).getProperties();
+        
+        File includedPropertiesFile = new File(testFolder.getRoot(), "included.properties");
+        Properties includedProperties = new Properties();
+        includedProperties.setProperty(includedKey, "irrelevantValue");
+        includedProperties.store(new FileWriter(includedPropertiesFile), null);
+        
+        mojo.setIncludePropertyKeysFromFiles(new String[] {includedPropertiesFile.getAbsolutePath()});
+        
+        // execute
+        mojo.execute();
+        
+        // assert
+        assertTrue(mojo.outputFile.exists());
+        Properties storedProperties = getStoredProperties();
+        assertEquals(1, storedProperties.size());
+        assertTrue(storedProperties.containsKey(includedKey));
+        assertEquals(includedValue, storedProperties.getProperty(includedKey));
+    }
+    
+    @Test
+    public void testExecuteIncludingPropertyKeysFromClasspathResource() throws Exception {
+        // given
+        String key = "projectPropertyKey";
+        String value = "projectPropertyValue";
+        String includedKey = "includedPropertyKey";
+        String includedValue = "includedPropertyValue";
+        Properties projectProperties = new Properties();
+        projectProperties.setProperty(key, value);
+        projectProperties.setProperty(includedKey, includedValue);
+        doReturn(projectProperties).when(mavenProject).getProperties();
+        
+        mojo.setIncludePropertyKeysFromFiles(new String[] {"/eu/dnetlib/maven/plugin/properties/included.properties"});
+        
+        // execute
+        mojo.execute();
+        
+        // assert
+        assertTrue(mojo.outputFile.exists());
+        Properties storedProperties = getStoredProperties();
+        assertEquals(1, storedProperties.size());
+        assertTrue(storedProperties.containsKey(includedKey));
+        assertEquals(includedValue, storedProperties.getProperty(includedKey));
+    }
+    
+    @Test(expected=MojoExecutionException.class)
+    public void testExecuteIncludingPropertyKeysFromBlankLocation() throws Exception {
+        // given
+        String key = "projectPropertyKey";
+        String value = "projectPropertyValue";
+        String includedKey = "includedPropertyKey";
+        String includedValue = "includedPropertyValue";
+        Properties projectProperties = new Properties();
+        projectProperties.setProperty(key, value);
+        projectProperties.setProperty(includedKey, includedValue);
+        doReturn(projectProperties).when(mavenProject).getProperties();
+        
+        mojo.setIncludePropertyKeysFromFiles(new String[] {""});
+        
+        // execute
+        mojo.execute();
+    }
+    
+    @Test
+    public void testExecuteIncludingPropertyKeysFromXmlFile() throws Exception {
+        // given
+        String key = "projectPropertyKey";
+        String value = "projectPropertyValue";
+        String includedKey = "includedPropertyKey";
+        String includedValue = "includedPropertyValue";
+        Properties projectProperties = new Properties();
+        projectProperties.setProperty(key, value);
+        projectProperties.setProperty(includedKey, includedValue);
+        doReturn(projectProperties).when(mavenProject).getProperties();
+        
+        File includedPropertiesFile = new File(testFolder.getRoot(), "included.xml");
+        Properties includedProperties = new Properties();
+        includedProperties.setProperty(includedKey, "irrelevantValue");
+        includedProperties.storeToXML(new FileOutputStream(includedPropertiesFile), null);
+        
+        mojo.setIncludePropertyKeysFromFiles(new String[] {includedPropertiesFile.getAbsolutePath()});
+        
+        // execute
+        mojo.execute();
+        
+        // assert
+        assertTrue(mojo.outputFile.exists());
+        Properties storedProperties = getStoredProperties();
+        assertEquals(1, storedProperties.size());
+        assertTrue(storedProperties.containsKey(includedKey));
+        assertEquals(includedValue, storedProperties.getProperty(includedKey));
+    }
+    
+    @Test(expected=MojoExecutionException.class)
+    public void testExecuteIncludingPropertyKeysFromInvalidXmlFile() throws Exception {
+        // given
+        String key = "projectPropertyKey";
+        String value = "projectPropertyValue";
+        String includedKey = "includedPropertyKey";
+        String includedValue = "includedPropertyValue";
+        Properties projectProperties = new Properties();
+        projectProperties.setProperty(key, value);
+        projectProperties.setProperty(includedKey, includedValue);
+        doReturn(projectProperties).when(mavenProject).getProperties();
+        
+        File includedPropertiesFile = new File(testFolder.getRoot(), "included.xml");
+        Properties includedProperties = new Properties();
+        includedProperties.setProperty(includedKey, "irrelevantValue");
+        includedProperties.store(new FileOutputStream(includedPropertiesFile), null);
+        
+        mojo.setIncludePropertyKeysFromFiles(new String[] {includedPropertiesFile.getAbsolutePath()});
+        
+        // execute
+        mojo.execute();
+    }
+    
+    @Test
+    public void testExecuteWithQuietModeOn() throws Exception {
+        // given
+        mojo.setQuiet(true);
+        mojo.setIncludePropertyKeysFromFiles(new String[] {"invalid location"});
+        
+        // execute
+        mojo.execute();
+        
+        // assert
+        assertTrue(mojo.outputFile.exists());
+        Properties storedProperties = getStoredProperties();
+        assertEquals(0, storedProperties.size());
+    }
+    
+    @Test(expected=MojoExecutionException.class)
+    public void testExecuteIncludingPropertyKeysFromInvalidFile() throws Exception {
+        // given
+        mojo.setIncludePropertyKeysFromFiles(new String[] {"invalid location"});
+        
+        // execute
+        mojo.execute();
+    }
+    
+    @Test
+    public void testExecuteWithEnvironmentProperties() throws Exception {
+        // given
+        mojo.setIncludeEnvironmentVariables(true);
+        
+        // execute
+        mojo.execute();
+        
+        // assert
+        assertTrue(mojo.outputFile.exists());
+        Properties storedProperties = getStoredProperties();
+        assertTrue(storedProperties.size() > 0);
+        for (Object currentKey : storedProperties.keySet()) {
+            assertTrue(((String)currentKey).startsWith(PROPERTY_PREFIX_ENV));
+        }
+    }
+    
+    @Test
+    public void testExecuteWithSystemProperties() throws Exception {
+        // given
+        String key = "systemPropertyKey";
+        String value = "systemPropertyValue";
+        System.setProperty(key, value);
+        mojo.setIncludeSystemProperties(true);
+        
+        // execute
+        mojo.execute();
+        
+        // assert
+        assertTrue(mojo.outputFile.exists());
+        Properties storedProperties = getStoredProperties();
+        assertTrue(storedProperties.size() > 0);
+        assertTrue(storedProperties.containsKey(key));
+        assertEquals(value, storedProperties.getProperty(key));
+    }
+    
+    @Test
+    public void testExecuteWithSystemPropertiesAndEscapeChars() throws Exception {
+        // given
+        String key = "systemPropertyKey ";
+        String value = "systemPropertyValue";
+        System.setProperty(key, value);
+        mojo.setIncludeSystemProperties(true);
+        String escapeChars = "cr,lf,tab,|";
+        mojo.setEscapeChars(escapeChars);
+        
+        // execute
+        mojo.execute();
+        
+        // assert
+        assertTrue(mojo.outputFile.exists());
+        Properties storedProperties = getStoredProperties();
+        assertTrue(storedProperties.size() > 0);
+        assertFalse(storedProperties.containsKey(key));
+        assertTrue(storedProperties.containsKey(key.trim()));
+        assertEquals(value, storedProperties.getProperty(key.trim()));
+    }
+    
+    // ----------------------------------- PRIVATE -------------------------------------------
+    
+    private File getPropertiesFileLocation() {
+        return new File(testFolder.getRoot(), "test.properties");
+    }
+    
+    private Properties getStoredProperties() throws FileNotFoundException, IOException {
+        Properties properties = new Properties();
+        properties.load(new FileInputStream(getPropertiesFileLocation()));
+        return properties;
+    }
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/CloseableIterator.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/CloseableIterator.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/CloseableIterator.java	(revision 51263)
@@ -0,0 +1,25 @@
+package eu.dnetlib.dhp.common.java.io;
+
+import java.io.Closeable;
+import java.util.Iterator;
+
+/**
+ * An iterator for I/O operations that can be {@code close}d explicitly to 
+ * release the resources it holds. 
+ * 
+ * You should call {@code close} only when interrupting the iteration in the 
+ * middle since in such situation there is no way for the iterator to know if 
+ * you're going to continue the iteration and it should still hold the resources 
+ * or not. There's no need to call {@code close} when iterating over all 
+ * elements since in such situation it is called automatically after the 
+ * end of iteration.
+ * 
+ * @author mhorst
+ *
+ * @param <E>
+ */
+public interface CloseableIterator<E> extends Iterator<E>, Closeable {
+
+	
+}
+
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/HdfsUtils.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/HdfsUtils.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/HdfsUtils.java	(revision 51263)
@@ -0,0 +1,37 @@
+package eu.dnetlib.dhp.common.java.io;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Util class containing operations on hdfs or local filesystem
+ * 
+ * @author madryk
+ */
+public final class HdfsUtils {
+
+    //------------------------ CONSTRUCTORS -------------------
+    
+    private HdfsUtils() {}
+    
+    //------------------------ LOGIC --------------------------
+    
+    /**
+     * Removes file or directory (recursively) located under the specified pathname.
+     */
+    public static void remove(Configuration hadoopConf, String pathname) throws IOException {
+        
+        Path path = new Path(pathname);
+        
+        FileSystem fileSystem = FileSystem.get(hadoopConf);
+        
+        if (fileSystem.exists(path)) {
+            fileSystem.delete(path, true);
+        }
+        
+    }
+    
+}
Index: dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/pom.xml
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/pom.xml	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/pom.xml	(revision 51263)
@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+	<modelVersion>4.0.0</modelVersion>
+
+	<parent>
+		<groupId>eu.dnetlib.dhp</groupId>
+		<artifactId>dhp-build</artifactId>
+		<version>1.0.0-SNAPSHOT</version>
+	</parent>
+
+	<artifactId>dhp-build-properties-maven-plugin</artifactId>
+	<packaging>maven-plugin</packaging>
+
+
+	<dependencies>
+		<dependency>
+			<groupId>org.apache.maven</groupId>
+			<artifactId>maven-plugin-api</artifactId>
+			<version>2.0</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.maven</groupId>
+			<artifactId>maven-project</artifactId>
+			<version>2.0</version>
+		</dependency>
+		<dependency>
+			<groupId>org.kuali.maven.plugins</groupId>
+			<artifactId>properties-maven-plugin</artifactId>
+			<version>1.3.2</version>
+		</dependency>
+	</dependencies>
+
+
+	<build>
+		<directory>target</directory>
+		<outputDirectory>target/classes</outputDirectory>
+		<finalName>${project.artifactId}-${project.version}</finalName>
+		<testOutputDirectory>target/test-classes</testOutputDirectory>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-compiler-plugin</artifactId>
+			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-source-plugin</artifactId>
+				<executions>
+					<execution>
+						<id>attach-sources</id>
+						<phase>verify</phase>
+						<goals>
+							<goal>jar-no-fork</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-javadoc-plugin</artifactId>
+				<configuration>
+					<detectLinks>true</detectLinks>
+				</configuration>
+			</plugin>
+		</plugins>
+	</build>
+
+</project>
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/DataStore.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/DataStore.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/DataStore.java	(revision 51263)
@@ -0,0 +1,172 @@
+package eu.dnetlib.dhp.common.java.io;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericContainer;
+import org.apache.avro.io.DatumWriter;
+import org.apache.avro.specific.SpecificDatumWriter;
+
+
+/**
+ * Utility for accessing to Avro-based data stores stored in file system
+ * @author Mateusz Kobos
+ *
+ */
+public final class DataStore {
+	
+	private final static String singleDataStoreFileName = "content.avro";
+	
+	private static final int FILE_NO_PADDING_LENGTH = 7;
+	
+	private DataStore(){}
+
+	/**
+	 * Create a new data store directory with single file and return writer that allows 
+	 * adding new records
+	 * @param path path to a directory to be created
+	 * @param schema schema of the records to be stored in the file
+	 * @return 
+	 * @throws IOException
+	 */
+	public static <T> DataFileWriter<T> create(
+			FileSystemPath path, Schema schema) throws IOException{
+		return create(path, schema, singleDataStoreFileName);
+	}
+	
+
+	/**
+     * Create a new data store directory and return writer that allows 
+     * adding new records
+     * @param path path to a directory to be created
+     * @param schema schema of the records to be stored in the file
+     * @param dataStoreFileName datastore file name
+     * @return 
+     * @throws IOException
+     */
+    public static <T> DataFileWriter<T> create(
+            FileSystemPath path, Schema schema, String dataStoreFileName) throws IOException{
+        path.getFileSystem().mkdirs(path.getPath());
+        FileSystemPath outFile = new FileSystemPath(
+                path, dataStoreFileName);
+        return DataStore.createSingleFile(outFile, schema);
+    }
+	
+	/**
+	 * Get reader for reading records from given data store
+	 * 
+	 * Here the schema used for reading the data store is set to be the same 
+	 * as the one that was used to write it.
+	 * 
+	 * @see getReader(FileSystemPath path, Schema readerSchema) for details.
+	 * 
+	 */
+	public static <T> CloseableIterator<T> getReader(FileSystemPath path) 
+			throws IOException{
+		return getReader(path, null);
+	}
+	
+	/**
+	 * Get reader for reading records from given data store
+	 * @param path path to a directory corresponding to data store
+	 * @param readerSchema the schema onto which the read data store will 
+	 * 	be projected
+	 */
+	public static <T> CloseableIterator<T> getReader(
+			FileSystemPath path, Schema readerSchema) throws IOException{
+		return new AvroDataStoreReader<T>(path, readerSchema);
+	}
+	
+	/**
+	 * Read data store entries and insert them into a list. A utility function.
+	 * 
+	 * Here the schema used for reading the data store is set to be the same 
+	 * as the one that was used to write it.
+	 */
+	public static <T> List<T> read(FileSystemPath path) 
+			throws IOException{
+		return read(path, null);
+	}
+	
+	/**
+	 * Read data store entries and insert them into a list. A utility function.
+	 * 
+	 * @param readerSchema the schema onto which the read data store will 
+	 * 	be projected
+	 */
+	public static <T> List<T> read(FileSystemPath path, Schema readerSchema) 
+			throws IOException{
+		CloseableIterator<T> iterator = getReader(path, readerSchema);
+		List<T> elems = new ArrayList<T>();
+		while(iterator.hasNext()){
+			elems.add(iterator.next());
+		}
+		return elems;
+	}
+	
+	/**
+	 * Create a data store from a list of entries. A utility function.
+	 * The schema is implicitly
+	 * taken from the first element from the {@code elements} list.
+	 * @param elements list of elements to write. At least one element has
+	 * to be present, because it is used to retrieve schema of the
+	 * structures passed in the list.
+	 */
+	public static <T extends GenericContainer> void create(
+			List<T> elements, FileSystemPath path) throws IOException{
+		if(elements.isEmpty()){
+			throw new IllegalArgumentException(
+					"The list of elements has to be non-empty");
+		}
+		Schema schema = elements.get(0).getSchema();
+		create(elements, path, schema);
+	}
+
+	/**
+	 * Create a data store from a list of entries with schema given explicitly.
+	 * A utility function.
+	 */
+	public static <T extends GenericContainer> void create(
+			List<T> elements, FileSystemPath path, Schema schema) 
+					throws IOException{
+		DataFileWriter<T> writer = create(path, schema);
+		try{
+			for(T i: elements){
+				writer.append(i);
+			}
+		} finally {
+			if(writer != null){
+				writer.close();
+			}
+		}
+	}
+	
+	/**
+	 * Create a single Avro file. This method shouldn't be normally used to
+	 * create data stores since it creates only a single Avro file, 
+	 * while a data store consists of a directory containing one or more files.
+	 */
+	public static <T> DataFileWriter<T> createSingleFile(
+			FileSystemPath path, Schema schema) throws IOException{
+		DatumWriter<T> datumWriter = new SpecificDatumWriter<T>();
+		DataFileWriter<T> writer = new DataFileWriter<T>(datumWriter);
+		writer.create(schema, path.getFileSystem().create(path.getPath()));
+		return writer;
+	}
+
+	/**
+	 * Generates filename for given file number.
+	 * @param fileNo file sequence number
+	 */
+	public static String generateFileName(int fileNo) {
+        StringBuffer strBuff = new StringBuffer(String.valueOf(fileNo));
+        while(strBuff.length()<FILE_NO_PADDING_LENGTH) {
+            strBuff.insert(0, '0');
+        }
+        strBuff.append(".avro");
+        return strBuff.toString();
+    }
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/SequenceFileTextValueReader.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/SequenceFileTextValueReader.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/io/SequenceFileTextValueReader.java	(revision 51263)
@@ -0,0 +1,159 @@
+package eu.dnetlib.dhp.common.java.io;
+
+import java.io.IOException;
+import java.util.NoSuchElementException;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.SequenceFile.Reader;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * Iterator that extracts sequence file's consecutive {@link Text} values.
+ * 
+ * @author mhorst
+ */
+public class SequenceFileTextValueReader implements CloseableIterator<Text> {
+
+	private SequenceFile.Reader sequenceReader;
+
+	private final RemoteIterator<LocatedFileStatus> fileIt;
+
+	private final FileSystem fs;
+
+	/**
+	 * Ignore file starting with underscore. Such files are also ignored by
+	 * default by map-reduce jobs.
+	 */
+	private final static Pattern WHITELIST_REGEXP = Pattern.compile("^[^_].*");
+
+	private Text toBeReturned;
+
+	//------------------------ CONSTRUCTORS --------------------------
+	
+	/**
+	 * Default constructor.
+	 * 
+	 * @param path HDFS path along with associated FileSystem
+	 * @throws IOException
+	 */
+	public SequenceFileTextValueReader(final FileSystemPath path) throws IOException {
+		this.fs = path.getFileSystem();
+		if (fs.isDirectory(path.getPath())) {
+			fileIt = fs.listFiles(path.getPath(), false);
+			sequenceReader = getNextSequenceReader();
+		} else {
+			fileIt = null;
+			sequenceReader = new Reader(fs.getConf(), SequenceFile.Reader.file(path.getPath()));
+		}
+	}
+
+	//------------------------ LOGIC ---------------------------------
+	
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see java.util.Iterator#hasNext()
+	 */
+	@Override
+	public boolean hasNext() {
+		// check and provide next when already returned
+		if (toBeReturned == null) {
+			toBeReturned = getNext();
+		}
+		return toBeReturned != null;
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see java.util.Iterator#next()
+	 */
+	@Override
+	public Text next() {
+		if (toBeReturned != null) {
+			// element fetched while executing hasNext()
+			Text result = toBeReturned;
+			toBeReturned = null;
+			return result;
+		} else {
+			Text resultCandidate = getNext();
+			if (resultCandidate!=null) {
+				return resultCandidate;
+			} else {
+				throw new NoSuchElementException();
+			}
+		}
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see eu.dnetlib.dhp.exp.iterator.ClosableIterator#close()
+	 */
+	@Override
+	public void close() throws IOException {
+		if (sequenceReader != null) {
+			sequenceReader.close();
+		}
+	}
+	
+	//------------------------ PRIVATE -------------------------------
+	
+	private final Reader getNextSequenceReader() throws IOException {
+		while (fileIt != null && fileIt.hasNext()) {
+			LocatedFileStatus currentFileStatus = fileIt.next();
+			if (isValidFile(currentFileStatus)) {
+				return new Reader(this.fs.getConf(), SequenceFile.Reader.file(currentFileStatus.getPath()));
+			}
+		}
+		// fallback
+		return null;
+	}
+
+	/**
+	 * Checks whether file is valid candidate.
+	 * 
+	 * @param fileStatus
+	 *            file status holding file name
+	 * @return true when valid, false otherwise
+	 */
+	private final boolean isValidFile(LocatedFileStatus fileStatus) {
+		if (fileStatus.isFile()) {
+			return WHITELIST_REGEXP.matcher(fileStatus.getPath().getName()).matches();
+		} else {
+			return false;
+		}
+	}
+
+	/**
+	 * @return next data package
+	 */
+	private Text getNext() {
+		try {
+			if (sequenceReader == null) {
+				return null;
+			}
+			Writable key = (Writable) ReflectionUtils.newInstance(sequenceReader.getKeyClass(), fs.getConf());
+			Writable value = (Writable) ReflectionUtils.newInstance(sequenceReader.getValueClass(), fs.getConf());
+			if (sequenceReader.next(key, value)) {
+				return (Text) value;
+			} else {
+				sequenceReader.close();
+				sequenceReader = getNextSequenceReader();
+				if (sequenceReader != null) {
+					return getNext();
+				}
+			}
+			// fallback
+			return null;
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+	}
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/CmdLineParserException.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/CmdLineParserException.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/CmdLineParserException.java	(revision 51263)
@@ -0,0 +1,21 @@
+package eu.dnetlib.dhp.common.java;
+
+/**
+ * Command line parsing exception
+ * @author Mateusz Kobos
+ *
+ */
+public class CmdLineParserException extends RuntimeException {
+	/**
+	 * 
+	 */
+	private static final long serialVersionUID = 9219928547611876284L;
+
+	public CmdLineParserException(String message){
+		super(message);
+	}
+	
+	public CmdLineParserException(String message, Throwable cause){
+		super(message, cause);
+	}
+}
Index: dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/README.markdown
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/README.markdown	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/README.markdown	(revision 51263)
@@ -0,0 +1,6 @@
+Maven plugin module utilized by `dhp-wf` for proper `job.properties` file building.
+
+It is based on http://site.kuali.org/maven/plugins/properties-maven-plugin/1.3.2/write-project-properties-mojo.html and supplemented with:
+
+* handling includePropertyKeysFromFiles property allowing writing only properties listed in given property files
+As a final outcome only properties listed in `<include>` element and listed as a keys in files from `<includePropertyKeysFromFiles>` element will be written to output file. 
Index: dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/get_working_dir.sh
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/get_working_dir.sh	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/get_working_dir.sh	(revision 51263)
@@ -0,0 +1,3 @@
+#!/bin/bash
+hadoop fs -get ${workingDir}
+
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/CmdLineParser.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/CmdLineParser.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/CmdLineParser.java	(revision 51263)
@@ -0,0 +1,98 @@
+package eu.dnetlib.dhp.common.java;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * 
+ * @author Mateusz Kobos
+ *
+ */
+@SuppressWarnings("deprecation")
+public final class CmdLineParser {
+	/** HACK: make the names of various types of parameters of the program 
+	* more readable, e.g. "--Input_person=..." instead of "-Iperson=...",
+	* "--Output_merged=..." instead of "-Omerged=...". I wasn't able to
+	* get such notation so far using the Apache CLI. */
+	public static final String constructorPrefix = "C";
+	public static final String inputPrefix = "I";
+	public static final String outputPrefix = "O";
+	public static final String specialParametersPrefix = "S";
+	/** HACK: This field should be removed since this list of special 
+	 * parameters is empty, thus not used anywhere.*/
+	public static final String[] mandatorySpecialParameters = new String[]{};
+	public static final String processParametersPrefix = "P";
+	
+	// ------------------------- CONSTRUCTORS ------------------------------
+	
+	private CmdLineParser() {}
+	
+	// ------------------------- LOGIC -------------------------------------
+	
+    public static CommandLine parse(String[] args) {
+		Options options = new Options();
+		@SuppressWarnings("static-access")
+		Option constructorParams = OptionBuilder.withArgName("STRING")
+				.hasArg()
+				.withDescription("Constructor parameter")
+				.withLongOpt("ConstructorParam")
+				.create(constructorPrefix);
+		options.addOption(constructorParams);
+		@SuppressWarnings("static-access")
+		Option inputs = OptionBuilder.withArgName("portName=URI")
+				.hasArgs(2)
+				.withValueSeparator()
+				.withDescription("Path binding for a given input port")
+				.withLongOpt("Input")
+				.create(inputPrefix);
+		options.addOption(inputs);
+		@SuppressWarnings("static-access")
+		Option outputs = OptionBuilder.withArgName("portName=URI")
+				.hasArgs(2)
+				.withValueSeparator()
+				.withDescription("Path binding for a given output port")
+				.create(outputPrefix);
+		options.addOption(outputs);
+		@SuppressWarnings("static-access")
+		Option specialParameter = OptionBuilder.withArgName("parameter_name=string")
+				.hasArgs(2)
+				.withValueSeparator()
+				.withDescription(String.format("Value of special parameter. "
+						+ "These are the mandatory parameters={%s}",
+						StringUtils.join(mandatorySpecialParameters, ",")))
+				.create(specialParametersPrefix);
+		options.addOption(specialParameter);
+		@SuppressWarnings("static-access")
+		Option otherParameter = OptionBuilder.withArgName("parameter_name=string")
+				.hasArgs(2)
+				.withValueSeparator()
+				.withDescription(
+						String.format("Value of some other parameter."))
+				.create(processParametersPrefix);
+		options.addOption(otherParameter);
+		
+		Option help = new Option("help", "print this message");
+		options.addOption(help);
+
+		CommandLineParser parser = new GnuParser();
+		try {
+			CommandLine cmdLine = parser.parse(options, args);
+			if(cmdLine.hasOption("help")){
+				HelpFormatter formatter = new HelpFormatter();
+				formatter.printHelp("", options );
+				System.exit(1);
+			}
+			return cmdLine;
+		} catch (ParseException e) {
+			throw new CmdLineParserException("Parsing command line arguments failed", e);
+		}
+		
+	}
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/jsonworkflownodes/ClassPathResourceToHdfsCopier.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/jsonworkflownodes/ClassPathResourceToHdfsCopier.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/jsonworkflownodes/ClassPathResourceToHdfsCopier.java	(revision 51263)
@@ -0,0 +1,54 @@
+package eu.dnetlib.dhp.common.java.jsonworkflownodes;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+
+import com.google.common.base.Preconditions;
+
+import eu.dnetlib.dhp.common.java.PortBindings;
+import eu.dnetlib.dhp.common.java.Process;
+import eu.dnetlib.dhp.common.java.porttype.PortType;
+
+/**
+ * Utility class responsible for copying resources available on classpath to specified HDFS location.
+ * @author mhorst
+ *
+ */
+public class ClassPathResourceToHdfsCopier implements Process {
+
+    private static final String PARAM_INPUT_CLASSPATH_RESOURCE = "inputClasspathResource";
+    
+    private static final String PARAM_OUTPUT_HDFS_FILE_LOCATION = "outputHdfsFileLocation";
+
+    @Override
+    public void run(PortBindings portBindings, Configuration conf, Map<String, String> parameters) throws Exception {
+        Preconditions.checkNotNull(parameters.get(PARAM_INPUT_CLASSPATH_RESOURCE), PARAM_INPUT_CLASSPATH_RESOURCE + " parameter was not specified!");
+        Preconditions.checkNotNull(parameters.get(PARAM_OUTPUT_HDFS_FILE_LOCATION), PARAM_OUTPUT_HDFS_FILE_LOCATION + " parameter was not specified!");
+
+        FileSystem fs = FileSystem.get(conf);
+        
+        try (InputStream in = Thread.currentThread().getContextClassLoader()
+                .getResourceAsStream(parameters.get(PARAM_INPUT_CLASSPATH_RESOURCE));
+                OutputStream os = fs.create(new Path(parameters.get(PARAM_OUTPUT_HDFS_FILE_LOCATION)))) {
+            IOUtils.copyBytes(in, os, 4096, false);
+        }
+    }
+
+    @Override
+    public Map<String, PortType> getInputPorts() {
+        return new HashMap<String, PortType>();
+    }
+
+    @Override
+    public Map<String, PortType> getOutputPorts() {
+        return new HashMap<String, PortType>();
+    }
+
+}
Index: dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/project-default.properties
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/project-default.properties	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/project-default.properties	(revision 51263)
@@ -0,0 +1,7 @@
+#sandboxName when not provided explicitly will be generated
+sandboxName=${sandboxName}
+sandboxDir=/user/${iis.hadoop.frontend.user.name}/${sandboxName}
+workingDir=${sandboxDir}/working_dir
+oozie.wf.application.path = ${nameNode}${sandboxDir}/${oozieAppDir}
+oozieTopWfApplicationPath = ${oozie.wf.application.path}
+
Index: dnet-hadoop/dhp-build/dhp-build-assembly-resources/dhp-build-assembly-resources.iml
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-assembly-resources/dhp-build-assembly-resources.iml	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-assembly-resources/dhp-build-assembly-resources.iml	(revision 51263)
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
+    <output url="file://$MODULE_DIR$/target/classes" />
+    <output-test url="file://$MODULE_DIR$/target/test-classes" />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
+      <excludeFolder url="file://$MODULE_DIR$/target" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.22" level="project" />
+    <orderEntry type="library" name="Maven: org.slf4j:slf4j-log4j12:1.7.22" level="project" />
+    <orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: javax.servlet:javax.servlet-api:3.1.0" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.12" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-all:1.3" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.mockito:mockito-all:1.10.19" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.powermock:powermock-core:1.6.6" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.powermock:powermock-reflect:1.6.6" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.objenesis:objenesis:2.4" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.javassist:javassist:3.21.0-GA" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.code.findbugs:annotations:3.0.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.jcip:jcip-annotations:1.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.code.findbugs:jsr305:3.0.1" level="project" />
+  </component>
+</module>
\ No newline at end of file
Index: dnet-hadoop/dhp-schemas/src/main/avro/eu/dnetlib/dhp/importer/NativeRecord.avdl
===================================================================
--- dnet-hadoop/dhp-schemas/src/main/avro/eu/dnetlib/dhp/importer/NativeRecord.avdl	(nonexistent)
+++ dnet-hadoop/dhp-schemas/src/main/avro/eu/dnetlib/dhp/importer/NativeRecord.avdl	(revision 51263)
@@ -0,0 +1,21 @@
+@namespace("eu.dnetlib.dhp.importer.schemas")
+protocol DHP {
+
+  enum RecordFormat {
+    XML, JSON
+  }
+
+  record ImportedRecord {
+
+    // record identifier
+    string id;
+
+    RecordFormat format;
+
+    // format name (OAF, OAI_DC, Datacite, etc) for which there is a parser implementation
+    string formatName;
+
+    // record body
+    string body;
+  }
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/jsonworkflownodes/StringPortSpecificationExtractor.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/jsonworkflownodes/StringPortSpecificationExtractor.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/java/jsonworkflownodes/StringPortSpecificationExtractor.java	(revision 51263)
@@ -0,0 +1,89 @@
+package eu.dnetlib.dhp.common.java.jsonworkflownodes;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Extracts information about port name and its properties from a string
+ * of a form "{port_name, property_1, property_2, ...}"
+ * @author Mateusz Kobos
+ */
+public class StringPortSpecificationExtractor {
+	private final String[] propertiesRegexp;
+	private final String portSpecificationRegexp;
+	private final Pattern pattern;
+
+	public static class PortSpecification {
+		
+	    private final String name;
+	    
+		private final String[] properties;
+		
+		public PortSpecification(String name, String[] properties) {
+			this.name = name;
+			this.properties = properties;
+		}
+
+        public String getName() {
+            return name;
+        }
+
+        public String[] getProperties() {
+            return properties;
+        }
+	}
+	
+	/**
+	 * @param propertiesRegexp regular expressions specifying pattern for 
+	 * each of the properties associated with a port. An example of a single 
+	 * specification: {@code "[\\w\\.]+"}.
+	 */
+	public StringPortSpecificationExtractor(String[] propertiesRegexp){
+		this.propertiesRegexp = propertiesRegexp;
+		this.portSpecificationRegexp = createRegexpString("[\\w\\._]+", propertiesRegexp);
+		this.pattern = Pattern.compile(this.portSpecificationRegexp);
+	}
+
+	private static String createRegexpString(String portNameRegexp, String[] propertiesRegexp){
+		StringBuilder regexp = new StringBuilder();
+		regexp.append("s*\\{\\s*");
+		regexp.append("("+portNameRegexp+")");
+		for(String propertyRegexp: propertiesRegexp){
+			regexp.append(",\\s*("+propertyRegexp+")");
+		}
+		regexp.append("\\s*\\}\\s*");
+		return regexp.toString();
+	}
+	
+	private int getPropertiesCount(){
+		return propertiesRegexp.length;
+	}
+	
+	public PortSpecification getSpecification(String text){
+		Matcher m = pattern.matcher(text);
+		if(!m.matches()){
+			throw new RuntimeException(String.format("Specification of " +
+					"the port (\"%s\") does not match regexp \"%s\"", 
+					text, portSpecificationRegexp));
+		}
+        final int expectedGroupsCount = getPropertiesCount()+1;
+		if(m.groupCount() != expectedGroupsCount){
+			StringBuilder groups = new StringBuilder();
+			for(int i = 0; i < m.groupCount(); i++){
+				groups.append("\""+m.group(i)+"\"");
+				if(i != m.groupCount()-1) {
+				    groups.append(", ");
+				}
+			}
+			throw new RuntimeException(String.format(
+				"Invalid output port specification \"%s\": got %d groups "+
+				"instead of %d (namely: %s)", text, m.groupCount(), 
+				expectedGroupsCount, groups.toString()));
+		}
+		String[] properties = new String[getPropertiesCount()];
+		for(int i = 0; i < getPropertiesCount(); i++){
+			properties[i] = m.group(i+2);
+		}
+		return new PortSpecification(m.group(1), properties);
+	}
+}
Index: dnet-hadoop/dhp-schemas/README.md
===================================================================
--- dnet-hadoop/dhp-schemas/README.md	(nonexistent)
+++ dnet-hadoop/dhp-schemas/README.md	(revision 51263)
@@ -0,0 +1,3 @@
+Description of the project
+--------------------------
+This project defines **serialization schemas** of Avro data store files that are used to pass data between workflow nodes in the system.
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacade.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacade.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceMDStoreFacade.java	(revision 51263)
@@ -0,0 +1,52 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import eu.dnetlib.data.mdstore.MDStoreService;
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
+import eu.dnetlib.enabling.tools.JaxwsServiceResolverImpl;
+
+/**
+ * WebService based MDStore facade.
+ * 
+ * @author mhorst
+ *
+ */
+public class WebServiceMDStoreFacade extends AbstractResultSetAwareWebServiceFacade<MDStoreService> implements MDStoreFacade {
+
+    
+    //------------------------ CONSTRUCTORS -------------------
+    
+    /**
+     * @param serviceLocation MDStore webservice location
+     * @param serviceReadTimeout service read timeout
+     * @param serviceConnectionTimeout service connection timeout
+     * @param resultSetReadTimeout resultset read timeout
+     * @param resultSetConnectionTimeout result set connection timeout
+     * @param resultSetPageSize resultset page size
+     */
+    public WebServiceMDStoreFacade(String serviceLocation, 
+            long serviceReadTimeout, long serviceConnectionTimeout,
+            long resultSetReadTimeout, long resultSetConnectionTimeout, int resultSetPageSize) {
+        super(MDStoreService.class, serviceLocation, 
+                serviceReadTimeout, serviceConnectionTimeout, 
+                resultSetReadTimeout, resultSetConnectionTimeout, resultSetPageSize);
+    }
+    
+    //------------------------ LOGIC --------------------------
+    
+    @Override
+    public Iterable<String> deliverMDRecords(String mdStoreId) throws ServiceFacadeException {
+        try {
+            W3CEndpointReference eprResult = getService().deliverMDRecords(mdStoreId, null, null, null);
+            ResultSetClientFactory rsFactory = new ResultSetClientFactory(
+                    getResultSetPageSize(), getResultSetReadTimeout(), getResultSetConnectionTimeout());
+            rsFactory.setServiceResolver(new JaxwsServiceResolverImpl());
+            return rsFactory.getClient(eprResult);
+        } catch (MDStoreServiceException e) {
+            throw new ServiceFacadeException("delivering records for md store " + mdStoreId + " failed!", e);
+        }
+    }
+
+}
Index: dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/print_working_dir.sh
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/print_working_dir.sh	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/print_working_dir.sh	(revision 51263)
@@ -0,0 +1,5 @@
+#!/bin/bash
+echo ""
+echo "---->Contents of the working directory"
+hadoop fs -ls ${workingDir}
+
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeException.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeException.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeException.java	(revision 51263)
@@ -0,0 +1,27 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+/**
+ * Service facade generic exception. 
+ * 
+ * @author mhorst
+ *
+ */
+public class ServiceFacadeException extends Exception {
+
+    private static final long serialVersionUID = 0L;
+
+    //------------------------ CONSTRUCTORS -------------------
+
+    public ServiceFacadeException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public ServiceFacadeException(String message) {
+        super(message);
+    }
+
+    public ServiceFacadeException(Throwable cause) {
+        super(cause);
+    }
+
+}
Index: dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/readme.markdown
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/readme.markdown	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/readme.markdown	(revision 51263)
@@ -0,0 +1,5 @@
+Execute the scripts in the following order:
+
+1. `upload_workflow.sh`
+2. `run_workflow.sh`
+3. `print_working_dir.sh` or `get_working_dir.sh`
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacade.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacade.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceISLookupFacade.java	(revision 51263)
@@ -0,0 +1,55 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+import java.util.Collections;
+
+import org.apache.log4j.Logger;
+
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpDocumentNotFoundException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+
+/**
+ * WebService based database facade.
+ * 
+ * @author mhorst
+ *
+ */
+public class WebServiceISLookupFacade extends AbstractResultSetAwareWebServiceFacade<ISLookUpService> implements ISLookupFacade {
+
+    private static final Logger log = Logger.getLogger(WebServiceISLookupFacade.class);
+
+    
+    //------------------------ CONSTRUCTORS -------------------
+    
+    /**
+     * @param serviceLocation database service location
+     * @param serviceReadTimeout service read timeout
+     * @param serviceConnectionTimeout service connection timeout
+     * @param resultSetReadTimeout result set providing database results read timeout
+     * @param resultSetConnectionTimeout result set connection timeout
+     * @param resultSetPageSize result set data chunk size
+     */
+    public WebServiceISLookupFacade(String serviceLocation, 
+            long serviceReadTimeout, long serviceConnectionTimeout,
+            long resultSetReadTimeout, long resultSetConnectionTimeout, int resultSetPageSize) {
+        super(ISLookUpService.class, serviceLocation, 
+                serviceReadTimeout, serviceConnectionTimeout, 
+                resultSetReadTimeout, resultSetConnectionTimeout, resultSetPageSize);
+    }
+
+    //------------------------ LOGIC --------------------------
+    
+    @Override
+    public Iterable<String> searchProfile(String xPathQuery) throws ServiceFacadeException {
+        try {
+            return getService().quickSearchProfile(xPathQuery);
+        }  catch (ISLookUpDocumentNotFoundException e) {
+            log.error("unable to find profile for query: " + xPathQuery, e);
+            return Collections.emptyList();
+        } catch (ISLookUpException e) {
+            throw new ServiceFacadeException("searching profiles in ISLookup failed with query '" + xPathQuery + "'", e);
+        }
+        
+    }
+
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/WorkflowRuntimeParameters.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/WorkflowRuntimeParameters.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/WorkflowRuntimeParameters.java	(revision 51263)
@@ -0,0 +1,74 @@
+package eu.dnetlib.dhp.common;
+
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Utility class holding parameter names and method simplifying access to parameters from hadoop context.
+ * @author mhorst
+ *
+ */
+public final class WorkflowRuntimeParameters {
+
+    public static final String OOZIE_ACTION_OUTPUT_FILENAME = "oozie.action.output.properties";
+    
+	public static final char DEFAULT_CSV_DELIMITER = ',';
+	
+	public static final String UNDEFINED_NONEMPTY_VALUE = "$UNDEFINED$";
+	
+    // default values
+    public static final String DNET_SERVICE_READ_TIMEOUT_DEFAULT_VALUE = "60000";
+    public static final String DNET_SERVICE_CONNECTION_TIMEOUT_DEFAULT_VALUE = "60000";
+    // parameter names
+    public static final String DNET_SERVICE_CLIENT_READ_TIMEOUT = "dnet.service.client.read.timeout";
+    public static final String DNET_SERVICE_CLIENT_CONNECTION_TIMEOUT = "dnet.service.client.connection.timeout";
+
+    // ----------------- CONSTRUCTORS -----------------------------
+    
+    private WorkflowRuntimeParameters() {}
+    
+	/**
+     * Retrieves parameter from hadoop context configuration when set to value different than {@link WorkflowRuntimeParameters#UNDEFINED_NONEMPTY_VALUE}.
+     */
+    public static String getParamValue(String paramName, Configuration configuration) {
+        String paramValue = configuration.get(paramName);
+        if (StringUtils.isNotBlank(paramValue) && !UNDEFINED_NONEMPTY_VALUE.equals(paramValue)) {
+            return paramValue;
+        } else {
+            return null;
+        }
+    }
+	
+    /**
+     * Retrieves {@link Integer} parameter from hadoop context configuration when set to non-empty value different than {@link WorkflowRuntimeParameters#UNDEFINED_NONEMPTY_VALUE}.
+     * Null is returned when parameter was not set.
+     * @throws {@link NumberFormatException} if parameter value does not contain a parsable integer
+     */
+    public static Integer getIntegerParamValue(String paramName, Configuration configuration) throws NumberFormatException {
+        String paramValue = getParamValue(paramName, configuration);
+        return paramValue!=null?Integer.valueOf(paramValue):null;
+    }
+    
+    /**
+     * Retrieves parameter from hadoop context configuration when set to value different than {@link WorkflowRuntimeParameters#UNDEFINED_NONEMPTY_VALUE}.
+     * If requested parameter was not set, fallback parameter is retrieved using the same logic.
+     */
+    public static String getParamValue(String paramName, String fallbackParamName, Configuration configuration) {
+        String resultCandidate = getParamValue(paramName, configuration);
+        return resultCandidate!=null?resultCandidate:getParamValue(fallbackParamName, configuration);
+    }
+    
+    /**
+     * Provides parameter value. Returns default value when entry not found among parameters.
+     * 
+     * @param paramName parameter name
+     * @param defaultValue parameter default value to be returned when entry not found among parameters
+     * @param parameters map of parameters
+     */
+    public static String getParamValue(String paramName, String defaultValue, Map<String, String> parameters) {
+        return parameters.containsKey(paramName)?parameters.get(paramName):defaultValue;
+    }
+    
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/oozie/OozieClientFactory.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/oozie/OozieClientFactory.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/oozie/OozieClientFactory.java	(revision 51263)
@@ -0,0 +1,24 @@
+package eu.dnetlib.dhp.common.oozie;
+
+import org.apache.oozie.client.OozieClient;
+
+/**
+ * Factory of {@link OozieClient}
+ * 
+ * @author madryk
+ */
+public class OozieClientFactory {
+
+    
+    //------------------------ LOGIC --------------------------
+    
+    /**
+     * Returns {@link OozieClient} object used for communication with oozie
+     */
+    public OozieClient createOozieClient(String oozieUrl) {
+        
+        OozieClient oozieClient = new OozieClient(oozieUrl);
+        
+        return oozieClient;
+    }
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/StringNormalizer.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/StringNormalizer.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/string/StringNormalizer.java	(revision 51263)
@@ -0,0 +1,16 @@
+package eu.dnetlib.dhp.common.string;
+
+/**
+ * String normalizer.
+ *
+ * @author Łukasz Dumiszewski
+ *
+ */
+public interface StringNormalizer {
+    
+    /**
+     * Normalizes the given string value.
+     */
+    String normalize(String value);
+    
+}
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacadeFactory.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacadeFactory.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/WebServiceObjectStoreFacadeFactory.java	(revision 51263)
@@ -0,0 +1,44 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_CLIENT_CONNECTION_TIMEOUT;
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_CLIENT_READ_TIMEOUT;
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_CONNECTION_TIMEOUT_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.common.WorkflowRuntimeParameters.DNET_SERVICE_READ_TIMEOUT_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_CONTENT_OBJECT_STORE_LOC;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_RESULT_SET_CLIENT_CONNECTION_TIMEOUT;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_RESULT_SET_CLIENT_READ_TIMEOUT;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.IMPORT_RESULT_SET_PAGESIZE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.RESULTSET_CONNECTION_TIMEOUT_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.RESULTSET_PAGESIZE_DEFAULT_VALUE;
+import static eu.dnetlib.dhp.wf.importer.ImportWorkflowRuntimeParameters.RESULTSET_READ_TIMEOUT_DEFAULT_VALUE;
+
+import java.util.Map;
+
+import com.google.common.base.Preconditions;
+
+import eu.dnetlib.dhp.common.WorkflowRuntimeParameters;
+
+/**
+ * WebService ObjectStore facade factory.
+ * 
+ * @author mhorst
+ *
+ */
+public class WebServiceObjectStoreFacadeFactory implements ServiceFacadeFactory<ObjectStoreFacade> {
+
+    //------------------------ LOGIC --------------------------
+    
+    @Override
+    public WebServiceObjectStoreFacade instantiate(Map<String, String> parameters) {
+        Preconditions.checkArgument(parameters.containsKey(IMPORT_CONTENT_OBJECT_STORE_LOC), 
+                "unknown object store service location: no parameter provided:  '%s'", IMPORT_CONTENT_OBJECT_STORE_LOC);
+
+        return new WebServiceObjectStoreFacade(parameters.get(IMPORT_CONTENT_OBJECT_STORE_LOC),
+                Long.parseLong(WorkflowRuntimeParameters.getParamValue(DNET_SERVICE_CLIENT_READ_TIMEOUT, DNET_SERVICE_READ_TIMEOUT_DEFAULT_VALUE, parameters)),
+                Long.parseLong(WorkflowRuntimeParameters.getParamValue(DNET_SERVICE_CLIENT_CONNECTION_TIMEOUT, DNET_SERVICE_CONNECTION_TIMEOUT_DEFAULT_VALUE, parameters)),
+                Long.parseLong(WorkflowRuntimeParameters.getParamValue(IMPORT_RESULT_SET_CLIENT_READ_TIMEOUT, RESULTSET_READ_TIMEOUT_DEFAULT_VALUE, parameters)),
+                Long.parseLong(WorkflowRuntimeParameters.getParamValue(IMPORT_RESULT_SET_CLIENT_CONNECTION_TIMEOUT, RESULTSET_CONNECTION_TIMEOUT_DEFAULT_VALUE, parameters)),
+                Integer.parseInt(WorkflowRuntimeParameters.getParamValue(IMPORT_RESULT_SET_PAGESIZE, RESULTSET_PAGESIZE_DEFAULT_VALUE, parameters)));
+    }
+
+}
Index: dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/report/ReportGenerator.java
===================================================================
--- dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/report/ReportGenerator.java	(nonexistent)
+++ dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/report/ReportGenerator.java	(revision 51263)
@@ -0,0 +1,110 @@
+package eu.dnetlib.dhp.common.report;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import com.google.common.collect.Lists;
+
+import eu.dnetlib.dhp.common.java.PortBindings;
+import eu.dnetlib.dhp.common.java.Process;
+import eu.dnetlib.dhp.common.java.io.DataStore;
+import eu.dnetlib.dhp.common.java.io.FileSystemPath;
+import eu.dnetlib.dhp.common.java.porttype.AvroPortType;
+import eu.dnetlib.dhp.common.java.porttype.PortType;
+import eu.dnetlib.dhp.common.schemas.ReportEntry;
+
+/**
+ * Java workflow node process for building report.<br/>
+ * It writes report properties into avro datastore of {@link ReportEntry}s
+ * with location specified in output port.<br/>
+ * Report property name must start with <code>report.</code> to
+ * be included in output datastore.
+ * 
+ * Usage example:<br/>
+ * <pre>
+ * {@code
+ * <action name="report">
+ *     <java>
+ *         <main-class>eu.dnetlib.dhp.common.java.ProcessWrapper</main-class>
+ *         <arg>eu.dnetlib.dhp.common.report.ReportGenerator</arg>
+ *         <arg>-Preport.someProperty=someValue</arg>
+ *         <arg>-Oreport=/report/path</arg>
+ *     </java>
+ *     ...
+ * </action>
+ * }
+ * </pre>
+ * Above example will produce avro datastore in <code>/report/path</code>
+ * with single {@link ReportEntry}.
+ * Where the {@link ReportEntry#getKey()} will be equal to <code>someProperty</code> and 
+ * the {@link ReportEntry#getValue()} will be equal to <code>someValue</code>
+ * (notice the stripped <code>report.</code> prefix from the entry key).
+ * 
+ * 
+ * @author madryk
+ *
+ */
+public class ReportGenerator implements Process {
+
+    private static final String REPORT_PORT_OUT_NAME = "report";
+    
+    private static final String REPORT_PROPERTY_PREFIX = "report.";
+    
+
+    //------------------------ LOGIC --------------------------
+    
+    @Override
+    public Map<String, PortType> getInputPorts() {
+        return Collections.emptyMap();
+    }
+
+    @Override
+    public Map<String, PortType> getOutputPorts() {
+        return Collections.singletonMap(REPORT_PORT_OUT_NAME, new AvroPortType(ReportEntry.SCHEMA$));
+    }
+    
+    @Override
+    public void run(PortBindings portBindings, Configuration conf, Map<String, String> parameters) throws Exception {
+        
+        Map<String, String> entriesToReport = collectEntriesToReport(parameters);
+        
+        List<ReportEntry> avroReport = convertToAvroReport(entriesToReport);
+        
+        
+        FileSystem fs = FileSystem.get(conf);
+        
+        Path reportPath = portBindings.getOutput().get(REPORT_PORT_OUT_NAME);
+        
+        DataStore.create(avroReport, new FileSystemPath(fs, reportPath));
+        
+    }
+    
+    
+    //------------------------ PRIVATE --------------------------
+    
+    private Map<String, String> collectEntriesToReport(Map<String, String> parameters) {
+        
+        return parameters.entrySet().stream()
+                .filter(property -> property.getKey().startsWith(REPORT_PROPERTY_PREFIX))
+                .map(x -> Pair.of(x.getKey().substring(REPORT_PROPERTY_PREFIX.length()), x.getValue()))
+                .collect(Collectors.toMap(e -> e.getLeft(), e -> e.getRight()));
+        
+    }
+
+    private List<ReportEntry> convertToAvroReport(Map<String, String> entriesToReport) {
+        
+        List<ReportEntry> avroReport = Lists.newArrayList();
+        entriesToReport.forEach((key, value) -> avroReport.add(ReportEntryFactory.createCounterReportEntry(key, Long.valueOf(value))));
+        
+        return avroReport;
+    }
+    
+
+}
Index: dnet-hadoop/dhp-common
===================================================================
--- dnet-hadoop/dhp-common	(nonexistent)
+++ dnet-hadoop/dhp-common	(revision 51263)

Property changes on: dnet-hadoop/dhp-common
___________________________________________________________________
Added: svn:ignore
## -0,0 +1,7 ##
+*~
+*.iml
+.idea
+.classpath
+.project
+.settings
+target
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/ImportWorkflowRuntimeParameters.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/ImportWorkflowRuntimeParameters.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/ImportWorkflowRuntimeParameters.java	(revision 51263)
@@ -0,0 +1,52 @@
+package eu.dnetlib.dhp.wf.importer;
+
+/**
+ * Import realated workflow parameters.
+ * @author mhorst
+ *
+ */
+public final class ImportWorkflowRuntimeParameters {
+	
+	// parameter names
+	
+	public static final String IMPORT_INFERENCE_PROVENANCE_BLACKLIST = "import.inference.provenance.blacklist";
+	public static final String IMPORT_SKIP_DELETED_BY_INFERENCE = "import.skip.deleted.by.inference";
+	public static final String IMPORT_TRUST_LEVEL_THRESHOLD = "import.trust.level.threshold";
+	public static final String IMPORT_APPROVED_DATASOURCES_CSV = "import.approved.datasources.csv";
+	public static final String IMPORT_APPROVED_COLUMNFAMILIES_CSV = "import.approved.columnfamilies.csv";
+	public static final String IMPORT_MERGE_BODY_WITH_UPDATES = "import.merge.body.with.updates";
+	public static final String IMPORT_CONTENT_APPROVED_OBJECSTORES_CSV = "import.content.approved.objectstores.csv";
+	public static final String IMPORT_CONTENT_BLACKLISTED_OBJECSTORES_CSV = "import.content.blacklisted.objectstores.csv";
+	
+	public static final String IMPORT_CONTENT_OBJECT_STORE_LOC = "import.content.object.store.location";
+	public static final String IMPORT_CONTENT_OBJECT_STORE_IDS_CSV = "import.content.object.store.ids.csv";
+	public static final String IMPORT_CONTENT_MAX_FILE_SIZE_MB = "import.content.max.file.size.mb";
+	public static final String IMPORT_CONTENT_CONNECTION_TIMEOUT = "import.content.connection.timeout";
+	public static final String IMPORT_CONTENT_READ_TIMEOUT = "import.content.read.timeout";
+
+	public static final String IMPORT_MDSTORE_IDS_CSV = "import.mdstore.ids.csv";
+	public static final String IMPORT_MDSTORE_SERVICE_LOCATION = "import.mdstore.service.location";
+	public static final String IMPORT_MDSTORE_RECORD_MAXLENGTH = "import.mdstore.record.maxlength";
+	
+	public static final String IMPORT_ISLOOKUP_SERVICE_LOCATION = "import.islookup.service.location";
+	public static final String IMPORT_VOCABULARY_CODE = "import.vocabulary.code";
+	public static final String IMPORT_VOCABULARY_OUTPUT_FILENAME = "import.vocabulary.output.filename";
+	
+	public static final String IMPORT_RESULT_SET_CLIENT_READ_TIMEOUT = "import.resultset.client.read.timeout";
+	public static final String IMPORT_RESULT_SET_CLIENT_CONNECTION_TIMEOUT = "import.resultset.client.connection.timeout";
+	public static final String IMPORT_RESULT_SET_PAGESIZE = "import.resultset.pagesize";
+	
+	
+	public static final String HBASE_ENCODING = "hbase.table.encoding";
+	
+	public static final String IMPORT_FACADE_FACTORY_CLASS = "import.facade.factory.classname";
+	
+	// default values
+	
+	public static final String RESULTSET_READ_TIMEOUT_DEFAULT_VALUE = "60000";
+	public static final String RESULTSET_CONNECTION_TIMEOUT_DEFAULT_VALUE = "60000";
+	public static final String RESULTSET_PAGESIZE_DEFAULT_VALUE = "100";
+	
+	private ImportWorkflowRuntimeParameters() {}
+	
+}
Index: dnet-hadoop/dhp-wf/dhp-wf.iml
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf.iml	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf.iml	(revision 51263)
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_5">
+    <output url="file://$MODULE_DIR$/target/classes" />
+    <output-test url="file://$MODULE_DIR$/target/test-classes" />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
+      <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
+      <excludeFolder url="file://$MODULE_DIR$/target" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="Maven: org.apache.oozie:oozie-client:4.1.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" name="Maven: com.googlecode.json-simple:json-simple:1.1.1" level="project" />
+    <orderEntry type="library" name="Maven: commons-cli:commons-cli:1.3.1" level="project" />
+    <orderEntry type="library" name="Maven: com.google.guava:guava:12.0" level="project" />
+    <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-core-asl:1.8.8" level="project" />
+    <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-mapper-asl:1.8.8" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.activemq:activemq-client:5.13.2" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.geronimo.specs:geronimo-jms_1.1_spec:1.1.1" level="project" />
+    <orderEntry type="library" name="Maven: org.fusesource.hawtbuf:hawtbuf:1.11" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.geronimo.specs:geronimo-j2ee-management_1.1_spec:1.0.1" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-auth:2.6.0-cdh5.9.0" level="project" />
+    <orderEntry type="library" name="Maven: commons-codec:commons-codec:1.4" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.2.5" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.2.4" level="project" />
+    <orderEntry type="library" name="Maven: commons-logging:commons-logging:1.1.1" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-i18n:2.0.0-M15" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.directory.api:api-asn1-api:1.0.0-M20" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.directory.api:api-util:1.0.0-M20" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.4.5-cdh5.9.0" level="project" />
+    <orderEntry type="library" name="Maven: jline:jline:2.11" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.curator:curator-framework:2.7.1" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.curator:curator-client:2.7.1" level="project" />
+    <orderEntry type="library" name="Maven: xerces:xercesImpl:2.10.0" level="project" />
+    <orderEntry type="library" name="Maven: xml-apis:xml-apis:1.4.01" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: net.schmizz:sshj:0.10.0" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.bouncycastle:bcpkix-jdk15on:1.50" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.bouncycastle:bcprov-jdk15on:1.50" level="project" />
+    <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.22" level="project" />
+    <orderEntry type="library" name="Maven: org.slf4j:slf4j-log4j12:1.7.22" level="project" />
+    <orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: javax.servlet:javax.servlet-api:3.1.0" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.12" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-all:1.3" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.mockito:mockito-all:1.10.19" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.powermock:powermock-core:1.6.6" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.powermock:powermock-reflect:1.6.6" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.objenesis:objenesis:2.4" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.javassist:javassist:3.21.0-GA" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.code.findbugs:annotations:3.0.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.jcip:jcip-annotations:1.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.code.findbugs:jsr305:3.0.1" level="project" />
+  </component>
+</module>
\ No newline at end of file
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MDRecordHandler.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MDRecordHandler.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/mdrecord/MDRecordHandler.java	(revision 51263)
@@ -0,0 +1,94 @@
+package eu.dnetlib.dhp.wf.importer.mdrecord;
+
+import java.util.Stack;
+
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+import eu.dnetlib.dhp.common.InfoSpaceConstants;
+
+/**
+ * MDRecord handler extracting record identifier.
+ * 
+ * Notice: writer is not being closed by handler. Created outside, let it be closed outside as well.
+ * @author mhorst
+ *
+ */
+public class MDRecordHandler extends DefaultHandler {
+
+    public static final String ELEM_OBJ_IDENTIFIER = "objIdentifier";
+    
+    private static final String ELEM_HEADER = "header";
+    
+    private Stack<String> parents;
+    
+    private StringBuilder currentValue = new StringBuilder();
+    
+    private String recordId;
+    
+    
+    // ------------------------ LOGIC --------------------------
+    
+    @Override
+    public void startDocument() throws SAXException {
+        parents = new Stack<String>();
+        recordId = null;
+    }
+
+    @Override
+    public void startElement(String uri, String localName, String qName,
+            Attributes attributes) throws SAXException {
+        if (this.recordId == null) {
+            if (isWithinElement(localName, ELEM_OBJ_IDENTIFIER, ELEM_HEADER)) {
+//              identifierType attribute is mandatory
+                this.currentValue = new StringBuilder();
+            }
+            this.parents.push(localName);            
+        }
+    }
+
+    @Override
+    public void endElement(String uri, String localName, String qName)
+            throws SAXException {
+        if (this.recordId == null) {
+            this.parents.pop();
+            if (isWithinElement(localName, ELEM_OBJ_IDENTIFIER, ELEM_HEADER)) {
+                this.recordId = InfoSpaceConstants.ROW_PREFIX_RESULT + this.currentValue.toString().trim();
+            }
+//          resetting current value;
+            this.currentValue = null;
+        }
+    }
+
+    @Override
+    public void endDocument() throws SAXException {
+        parents.clear();
+        parents = null;
+    }
+
+    @Override
+    public void characters(char[] ch, int start, int length)
+            throws SAXException {
+        if (this.currentValue!=null) {
+            this.currentValue.append(ch, start, length);
+        }
+    }
+
+    /**
+     * @return record identifier
+     */
+    public String getRecordId() {
+        return recordId;
+    }
+    
+    // ------------------------ PRIVATE --------------------------
+    
+    private boolean isWithinElement(String localName, String expectedElement, String expectedParent) {
+        return localName.equals(expectedElement) && !this.parents.isEmpty() && 
+                expectedParent.equals(this.parents.peek());
+    }
+
+    
+}
+
Index: dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/GenerateOoziePropertiesMojoTest.java
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/GenerateOoziePropertiesMojoTest.java	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/GenerateOoziePropertiesMojoTest.java	(revision 51263)
@@ -0,0 +1,101 @@
+package eu.dnetlib.maven.plugin.properties;
+
+import static eu.dnetlib.maven.plugin.properties.GenerateOoziePropertiesMojo.PROPERTY_NAME_SANDBOX_NAME;
+import static eu.dnetlib.maven.plugin.properties.GenerateOoziePropertiesMojo.PROPERTY_NAME_WF_SOURCE_DIR;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * @author mhorst
+ *
+ */
+public class GenerateOoziePropertiesMojoTest {
+
+    private GenerateOoziePropertiesMojo mojo = new GenerateOoziePropertiesMojo();
+    
+    @Before
+    public void clearSystemProperties() {
+        System.clearProperty(PROPERTY_NAME_SANDBOX_NAME);
+        System.clearProperty(PROPERTY_NAME_WF_SOURCE_DIR);
+    }
+    
+    @Test
+    public void testExecuteEmpty() throws Exception {
+        // execute
+        mojo.execute();
+        
+        // assert
+        assertNull(System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
+    }
+
+    @Test
+    public void testExecuteSandboxNameAlreadySet() throws Exception {
+        // given
+        String workflowSourceDir = "eu/dnetlib/iis/wf/transformers";
+        String sandboxName = "originalSandboxName";
+        System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
+        System.setProperty(PROPERTY_NAME_SANDBOX_NAME, sandboxName);
+        
+        // execute
+        mojo.execute();
+        
+        // assert
+        assertEquals(sandboxName, System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
+    }
+    
+    @Test
+    public void testExecuteEmptyWorkflowSourceDir() throws Exception {
+        // given
+        String workflowSourceDir = "";
+        System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
+        
+        // execute
+        mojo.execute();
+        
+        // assert
+        assertNull(System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
+    }
+    
+    @Test
+    public void testExecuteNullSandboxNameGenerated() throws Exception {
+        // given
+        String workflowSourceDir = "eu/dnetlib/iis/";
+        System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
+        
+        // execute
+        mojo.execute();
+        
+        // assert
+        assertNull(System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
+    }
+    
+    @Test
+    public void testExecute() throws Exception {
+        // given
+        String workflowSourceDir = "eu/dnetlib/iis/wf/transformers";
+        System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
+        
+        // execute
+        mojo.execute();
+        
+        // assert
+        assertEquals("wf/transformers", System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
+    }
+    
+    @Test
+    public void testExecuteWithoutRoot() throws Exception {
+        // given
+        String workflowSourceDir = "wf/transformers";
+        System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
+        
+        // execute
+        mojo.execute();
+        
+        // assert
+        assertEquals("wf/transformers", System.getProperty(PROPERTY_NAME_SANDBOX_NAME));
+    }
+    
+}
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/AbstractResultSetAwareWebServiceFacade.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/AbstractResultSetAwareWebServiceFacade.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/AbstractResultSetAwareWebServiceFacade.java	(revision 51263)
@@ -0,0 +1,104 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+import java.util.Map;
+
+import javax.xml.ws.BindingProvider;
+import javax.xml.ws.wsaddressing.W3CEndpointReferenceBuilder;
+
+import org.apache.log4j.Logger;
+
+import eu.dnetlib.enabling.tools.JaxwsServiceResolverImpl;
+
+/**
+ * Abstract class utilized by all WebService facades.
+ * @author mhorst
+ * 
+ */
+public abstract class AbstractResultSetAwareWebServiceFacade<T> {
+
+    private final Logger log = Logger.getLogger(this.getClass());
+    
+    /**
+     * Web service.
+     */
+    private final T service;
+    
+    /**
+     * ResultSet read timeout.
+     */
+    private final long resultSetReadTimeout;
+    
+    /**
+     * ResultSet connection timeout.
+     */
+    private final long resultSetConnectionTimeout;
+    
+    /**
+     * ResultSet page size.
+     */
+    private final int resultSetPageSize;
+    
+    
+    //------------------------ CONSTRUCTORS -------------------
+    
+    /**
+     * Instantiates underlying service.
+     * @param clazz webservice class
+     * @param serviceLocation webservice location
+     * @param serviceReadTimeout service read timeout
+     * @param serviceConnectionTimeout service connection timeout
+     * @param resultSetReadTimeout resultset read timeout
+     * @param resultSetConnectionTimeout resultset connection timeout
+     * @param resultSetPageSize resultset page size
+     */
+    protected AbstractResultSetAwareWebServiceFacade(Class<T> clazz, String serviceLocation,
+            long serviceReadTimeout, long serviceConnectionTimeout, 
+            long resultSetReadTimeout, long resultSetConnectionTimeout, int resultSetPageSize) {
+        W3CEndpointReferenceBuilder eprBuilder = new W3CEndpointReferenceBuilder();
+        eprBuilder.address(serviceLocation);
+        eprBuilder.build();
+        this.service = new JaxwsServiceResolverImpl().getService(clazz, eprBuilder.build());
+        if (this.service instanceof BindingProvider) {
+            log.info(String.format("setting timeouts for %s: read timeout (%s) and connect timeout (%s)", 
+                    BindingProvider.class, serviceReadTimeout, serviceConnectionTimeout));
+            final Map<String, Object> requestContext = ((BindingProvider) service).getRequestContext();
+
+            // can't be sure about which will be used. Set them all.
+            requestContext.put("com.sun.xml.internal.ws.request.timeout", serviceReadTimeout);
+            requestContext.put("com.sun.xml.internal.ws.connect.timeout", serviceConnectionTimeout);
+
+            requestContext.put("com.sun.xml.ws.request.timeout", serviceReadTimeout);
+            requestContext.put("com.sun.xml.ws.connect.timeout", serviceConnectionTimeout);
+
+            requestContext.put("javax.xml.ws.client.receiveTimeout", serviceReadTimeout);
+            requestContext.put("javax.xml.ws.client.connectionTimeout", serviceConnectionTimeout);
+        }
+        
+        this.resultSetReadTimeout = resultSetReadTimeout;
+        this.resultSetConnectionTimeout = resultSetConnectionTimeout;
+        this.resultSetPageSize = resultSetPageSize;
+    }
+
+    
+    //------------------------ GETTERS -------------------------
+
+    public T getService() {
+        return service;
+    }
+
+
+    public long getResultSetReadTimeout() {
+        return resultSetReadTimeout;
+    }
+
+
+    public long getResultSetConnectionTimeout() {
+        return resultSetConnectionTimeout;
+    }
+
+
+    public int getResultSetPageSize() {
+        return resultSetPageSize;
+    }
+    
+}
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ObjectStoreFacade.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ObjectStoreFacade.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ObjectStoreFacade.java	(revision 51263)
@@ -0,0 +1,19 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+/**
+ * ObjectStore service facade.
+ * 
+ * @author mhorst
+ *
+ */
+public interface ObjectStoreFacade {
+
+    /**
+     * Returns metadata records from given objectstore created in specified time range. 
+     * @param objectStoreId object store identifier
+     * @param from from time in millis
+     * @param until until time in millis
+     */
+    Iterable<String> deliverObjects(String objectStoreId, long from, long until) throws ServiceFacadeException;
+    
+}
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeFactory.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeFactory.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/facade/ServiceFacadeFactory.java	(revision 51263)
@@ -0,0 +1,20 @@
+package eu.dnetlib.dhp.wf.importer.facade;
+
+import java.util.Map;
+
+/**
+ * Generic service facade factory. All implementations must be instantiable with no-argument construtor.
+ * 
+ * @author mhorst
+ * 
+ */
+public interface ServiceFacadeFactory<T> {
+
+    /**
+     * Creates service of given type configured with parameters.
+     * 
+     * @param parameters service configuration
+     * 
+     */
+    T instantiate(Map<String, String> parameters);
+}
Index: dnet-hadoop/dhp-build/dhp-build-assembly-resources/pom.xml
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-assembly-resources/pom.xml	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-assembly-resources/pom.xml	(revision 51263)
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+	<modelVersion>4.0.0</modelVersion>
+
+	<parent>
+		<groupId>eu.dnetlib.dhp</groupId>
+		<artifactId>dhp-build</artifactId>
+		<version>1.0.0-SNAPSHOT</version>
+	</parent>
+
+	<artifactId>dhp-build-assembly-resources</artifactId>
+	<packaging>jar</packaging>
+
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-compiler-plugin</artifactId>
+			</plugin>
+		</plugins>
+	</build>
+
+</project>
Index: dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiver.java
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiver.java	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import/src/main/java/eu/dnetlib/dhp/wf/importer/DataFileRecordReceiver.java	(revision 51263)
@@ -0,0 +1,29 @@
+package eu.dnetlib.dhp.wf.importer;
+
+import java.io.IOException;
+
+import org.apache.avro.file.DataFileWriter;
+
+/**
+ * {@link DataFileWriter} based record receiver.
+ * @author mhorst
+ *
+ */
+public class DataFileRecordReceiver<T> implements RecordReceiver<T> {
+
+	private final DataFileWriter<T> writer;
+	
+	/**
+	 * Default constructor.
+	 * @param writer
+	 */
+	public DataFileRecordReceiver(DataFileWriter<T> writer) {
+		this.writer = writer;
+	}
+	
+	@Override
+	public void receive(T object) throws IOException {
+		this.writer.append(object);
+	}
+
+}
Index: dnet-hadoop/dhp-wf/dhp-wf-import
===================================================================
--- dnet-hadoop/dhp-wf/dhp-wf-import	(nonexistent)
+++ dnet-hadoop/dhp-wf/dhp-wf-import	(revision 51263)

Property changes on: dnet-hadoop/dhp-wf/dhp-wf-import
___________________________________________________________________
Added: svn:ignore
## -0,0 +1,7 ##
+*~
+*.iml
+.idea
+.classpath
+.project
+.settings
+target
Index: dnet-hadoop/dhp-wf
===================================================================
--- dnet-hadoop/dhp-wf	(nonexistent)
+++ dnet-hadoop/dhp-wf	(revision 51263)

Property changes on: dnet-hadoop/dhp-wf
___________________________________________________________________
Added: svn:ignore
## -0,0 +1,7 ##
+*~
+*.iml
+.idea
+.classpath
+.project
+.settings
+target
Index: dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/dhp-build-properties-maven-plugin.iml
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/dhp-build-properties-maven-plugin.iml	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/dhp-build-properties-maven-plugin.iml	(revision 51263)
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
+    <output url="file://$MODULE_DIR$/target/classes" />
+    <output-test url="file://$MODULE_DIR$/target/test-classes" />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
+      <sourceFolder url="file://$MODULE_DIR$/src/test/resources" type="java-test-resource" />
+      <excludeFolder url="file://$MODULE_DIR$/target" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="Maven: org.apache.maven:maven-plugin-api:2.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.maven:maven-project:2.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.maven:maven-profile:2.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.maven:maven-model:2.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.maven:maven-artifact-manager:2.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.maven:maven-repository-metadata:2.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.maven.wagon:wagon-provider-api:1.0-alpha-5" level="project" />
+    <orderEntry type="library" name="Maven: org.codehaus.plexus:plexus-utils:1.0.4" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.maven:maven-artifact:2.0" level="project" />
+    <orderEntry type="library" name="Maven: org.codehaus.plexus:plexus-container-default:1.0-alpha-8" level="project" />
+    <orderEntry type="library" name="Maven: classworlds:classworlds:1.1-alpha-2" level="project" />
+    <orderEntry type="library" name="Maven: org.kuali.maven.plugins:properties-maven-plugin:1.3.2" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework:spring-core:3.1.1.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: org.springframework:spring-asm:3.1.1.RELEASE" level="project" />
+    <orderEntry type="library" name="Maven: org.jasypt:jasypt:1.9.0" level="project" />
+    <orderEntry type="library" name="Maven: org.kuali.maven.common:maven-kuali-common:1.2.8" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.ant:ant:1.8.2" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.ant:ant-launcher:1.8.2" level="project" />
+    <orderEntry type="library" name="Maven: org.codehaus.plexus:plexus-interpolation:1.15" level="project" />
+    <orderEntry type="library" name="Maven: commons-lang:commons-lang:2.6" level="project" />
+    <orderEntry type="library" name="Maven: commons-io:commons-io:2.5" level="project" />
+    <orderEntry type="library" name="Maven: org.slf4j:jcl-over-slf4j:1.6.4" level="project" />
+    <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.22" level="project" />
+    <orderEntry type="library" name="Maven: org.slf4j:slf4j-log4j12:1.7.22" level="project" />
+    <orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: javax.servlet:javax.servlet-api:3.1.0" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.12" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-all:1.3" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.mockito:mockito-all:1.10.19" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.powermock:powermock-core:1.6.6" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.powermock:powermock-reflect:1.6.6" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.objenesis:objenesis:2.4" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.javassist:javassist:3.21.0-GA" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.code.findbugs:annotations:3.0.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.jcip:jcip-annotations:1.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.code.findbugs:jsr305:3.0.1" level="project" />
+  </component>
+</module>
\ No newline at end of file
Index: dnet-hadoop/dhp.iml
===================================================================
--- dnet-hadoop/dhp.iml	(nonexistent)
+++ dnet-hadoop/dhp.iml	(revision 51263)
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_5">
+    <output url="file://$MODULE_DIR$/target/classes" />
+    <output-test url="file://$MODULE_DIR$/target/test-classes" />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
+      <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
+      <excludeFolder url="file://$MODULE_DIR$/target" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.22" level="project" />
+    <orderEntry type="library" name="Maven: org.slf4j:slf4j-log4j12:1.7.22" level="project" />
+    <orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: javax.servlet:javax.servlet-api:3.1.0" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.12" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-all:1.3" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.mockito:mockito-all:1.10.19" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.powermock:powermock-core:1.6.6" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.powermock:powermock-reflect:1.6.6" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.objenesis:objenesis:2.4" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.javassist:javassist:3.21.0-GA" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.code.findbugs:annotations:3.0.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.jcip:jcip-annotations:1.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.code.findbugs:jsr305:3.0.1" level="project" />
+  </component>
+</module>
\ No newline at end of file
Index: dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/src/main/java/eu/dnetlib/maven/plugin/properties/GenerateOoziePropertiesMojo.java
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/src/main/java/eu/dnetlib/maven/plugin/properties/GenerateOoziePropertiesMojo.java	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/src/main/java/eu/dnetlib/maven/plugin/properties/GenerateOoziePropertiesMojo.java	(revision 51263)
@@ -0,0 +1,71 @@
+package eu.dnetlib.maven.plugin.properties;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.maven.plugin.AbstractMojo;
+import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugin.MojoFailureException;
+
+/**
+ * Generates oozie properties which were not provided from commandline.
+ * @author mhorst
+ *
+ * @goal generate-properties
+ */
+public class GenerateOoziePropertiesMojo extends AbstractMojo {
+
+	public static final String PROPERTY_NAME_WF_SOURCE_DIR = "workflow.source.dir";
+	public static final String PROPERTY_NAME_SANDBOX_NAME = "sandboxName";
+	
+	private final String[] limiters = {"iis", "dnetlib", "eu", "dhp"};
+	
+    @Override
+    public void execute() throws MojoExecutionException, MojoFailureException {
+    	if (System.getProperties().containsKey(PROPERTY_NAME_WF_SOURCE_DIR) &&
+    			!System.getProperties().containsKey(PROPERTY_NAME_SANDBOX_NAME)) {
+    		String generatedSandboxName = generateSandboxName(System.getProperties().getProperty(
+					PROPERTY_NAME_WF_SOURCE_DIR));
+    		if (generatedSandboxName!=null) {
+    			System.getProperties().setProperty(PROPERTY_NAME_SANDBOX_NAME, 
+        				generatedSandboxName);	
+    		} else {
+    			System.out.println("unable to generate sandbox name from path: " + 
+    					System.getProperties().getProperty(PROPERTY_NAME_WF_SOURCE_DIR));
+    		}
+    	}
+    }
+    
+    /**
+     * Generates sandbox name from workflow source directory.
+     * @param wfSourceDir
+     * @return generated sandbox name
+     */
+    private String generateSandboxName(String wfSourceDir) {
+//    	utilize all dir names until finding one of the limiters 
+    	List<String> sandboxNameParts = new ArrayList<String>(); 
+    	String[] tokens = StringUtils.split(wfSourceDir, File.separatorChar);
+    	ArrayUtils.reverse(tokens);
+    	if (tokens.length>0) {
+    		for (String token : tokens) {
+    			for (String limiter : limiters) {
+    				if (limiter.equals(token)) {
+    					return sandboxNameParts.size()>0?
+    							StringUtils.join(sandboxNameParts.toArray()):null;
+    				}
+    			}
+    			if (sandboxNameParts.size()>0) {
+    				sandboxNameParts.add(0, File.separator);
+    			}
+    			sandboxNameParts.add(0, token);
+    		}
+			return StringUtils.join(sandboxNameParts.toArray());	
+    	} else {
+    		return null;
+    	}
+    }
+
+}
Index: dnet-hadoop/dhp-schemas/pom.xml
===================================================================
--- dnet-hadoop/dhp-schemas/pom.xml	(nonexistent)
+++ dnet-hadoop/dhp-schemas/pom.xml	(revision 51263)
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>eu.dnetlib.dhp</groupId>
+        <artifactId>dhp</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>dhp-schemas</artifactId>
+    <packaging>jar</packaging>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.avro</groupId>
+            <artifactId>avro</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <!-- Plugin that generates Java classes from Avro schemas -->
+            <plugin>
+                <groupId>org.apache.avro</groupId>
+                <artifactId>avro-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>schema</goal>
+                            <goal>idl-protocol</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <!-- This plugin makes the Maven->Update Project Configuration
+			not forget about the "target/generated-sources/avro" source path-->
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>build-helper-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>add-source</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>add-source</goal>
+                        </goals>
+                        <configuration>
+                            <sources>
+                                <source>${project.build.directory}/generated-sources/avro/</source>
+                            </sources>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
Index: dnet-hadoop/dhp-build/dhp-build.iml
===================================================================
--- dnet-hadoop/dhp-build/dhp-build.iml	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build.iml	(revision 51263)
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_5">
+    <output url="file://$MODULE_DIR$/target/classes" />
+    <output-test url="file://$MODULE_DIR$/target/test-classes" />
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/target" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.22" level="project" />
+    <orderEntry type="library" name="Maven: org.slf4j:slf4j-log4j12:1.7.22" level="project" />
+    <orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: javax.servlet:javax.servlet-api:3.1.0" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.12" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-all:1.3" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.mockito:mockito-all:1.10.19" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.powermock:powermock-core:1.6.6" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.powermock:powermock-reflect:1.6.6" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.objenesis:objenesis:2.4" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.javassist:javassist:3.21.0-GA" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.code.findbugs:annotations:3.0.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.jcip:jcip-annotations:1.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.code.findbugs:jsr305:3.0.1" level="project" />
+  </component>
+</module>
\ No newline at end of file
Index: dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/run_workflow.sh
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/run_workflow.sh	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/run_workflow.sh	(revision 51263)
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+if [ $# = 0 ] ; then
+    oozie job -oozie ${oozieServiceLoc} -config job.properties -run
+else
+    oozie job -oozie ${oozieServiceLoc} -config $1/job.properties -run
+fi
+
+
+
Index: dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/assemblies/oozie-installer.xml
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/assemblies/oozie-installer.xml	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/assemblies/oozie-installer.xml	(revision 51263)
@@ -0,0 +1,32 @@
+<assembly
+	xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+	
+	<id>oozie-installer</id>
+	<formats>
+		<format>dir</format>
+	</formats>
+	
+	<fileSets>
+		<fileSet>
+			<filtered>true</filtered>
+			<directory>${project.build.directory}/assembly-resources/commands</directory>
+			<!-- 
+			dziala dla (lokalnie zasoby modulu):
+			<directory>src/main/resources</directory>
+			nie dziala dla:
+			<directory>classpath:/commands</directory>
+			<directory>commands</directory>
+			<directory>classpath/src/main/resources</directory>
+			 -->
+			<outputDirectory>/</outputDirectory>
+			<includes>
+				<include>**/*</include>
+			</includes>
+			<fileMode>0755</fileMode>
+			<lineEnding>unix</lineEnding>
+		</fileSet>
+	</fileSets>
+	<baseDirectory>/</baseDirectory>
+</assembly>
Index: dnet-hadoop/dhp-build/dhp-build-assembly-resources/README.markdown
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-assembly-resources/README.markdown	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-assembly-resources/README.markdown	(revision 51263)
@@ -0,0 +1,7 @@
+Module utilized by `dhp-wf`.
+
+Contains all required resources by this parent module:
+
+* assembly XML definitions
+* build shell scripts
+* oozie package commands for uploading, running and monitoring oozie workflows
Index: dnet-hadoop/dhp-schemas/src/main/avro/eu/dnetlib/dhp/audit/Fault.avdl
===================================================================
--- dnet-hadoop/dhp-schemas/src/main/avro/eu/dnetlib/dhp/audit/Fault.avdl	(nonexistent)
+++ dnet-hadoop/dhp-schemas/src/main/avro/eu/dnetlib/dhp/audit/Fault.avdl	(revision 51263)
@@ -0,0 +1,29 @@
+@namespace("eu.dnetlib.dhp.audit.schemas")
+protocol DHP {
+
+	record Cause {
+//		generic cause code, root exception class name when derived from exception         
+        string code;
+//		cause message
+		union { null , string } message = null;
+	}
+
+    record Fault {
+//      input object identifier
+        string inputObjectId;
+//		fault creation timestamp        
+        long timestamp;
+//      generic fault code, root exception class name when derived from exception         
+        string code;
+//		fault message
+		union { null , string } message = null;
+//		stack trace
+		union { null , string } stackTrace = null;
+//		fault causes, array is indexed with cause depth         
+        union { null , array<Cause> } causes = null;
+//		Other supplementary data related to specific type of fault. 
+//		See parameters description in oozie workflow.xml documentation of modules 
+//		that use this structure for information what exactly can be stored as supplementary data.
+		union { null , map<string> } supplementaryData = null;        
+    }
+}
Index: dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/src/main/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectProperties.java
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/src/main/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectProperties.java	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin/src/main/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectProperties.java	(revision 51263)
@@ -0,0 +1,436 @@
+/**
+ * 
+ * Licensed under the Educational Community License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.opensource.org/licenses/ecl2.php
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package eu.dnetlib.maven.plugin.properties;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.maven.plugin.AbstractMojo;
+import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugin.MojoFailureException;
+import org.apache.maven.project.MavenProject;
+import org.springframework.core.io.DefaultResourceLoader;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.ResourceLoader;
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
+/**
+ * Writes project properties for the keys listed in specified properties files.
+ * Based on: 
+ * http://site.kuali.org/maven/plugins/properties-maven-plugin/1.3.2/write-project-properties-mojo.html
+
+ * @author mhorst
+ * @goal write-project-properties
+ */
+public class WritePredefinedProjectProperties extends AbstractMojo {
+	
+	private static final String CR = "\r";
+	private static final String LF = "\n";
+	private static final String TAB = "\t";
+    protected static final String PROPERTY_PREFIX_ENV = "env.";
+    private static final String ENCODING_UTF8 = "utf8";
+	
+	/**
+	 * @parameter property="properties.includePropertyKeysFromFiles"
+	 */
+	private String[] includePropertyKeysFromFiles;
+	
+    /**
+     * @parameter default-value="${project}"
+     * @required
+     * @readonly
+     */
+    protected MavenProject project;
+
+    /**
+     * The file that properties will be written to
+     * 
+     * @parameter property="properties.outputFile"
+     *            default-value="${project.build.directory}/properties/project.properties";
+     * @required
+     */
+    protected File outputFile;
+	
+	/**
+     * If true, the plugin will silently ignore any non-existent properties files, and the build will continue
+     *
+     * @parameter property="properties.quiet" default-value="true"
+     */
+    private boolean quiet;
+	
+    /**
+     * Comma separated list of characters to escape when writing property values. cr=carriage return, lf=linefeed,
+     * tab=tab. Any other values are taken literally.
+     * 
+     * @parameter default-value="cr,lf,tab" property="properties.escapeChars"
+     */
+    private String escapeChars;
+
+    /**
+     * If true, the plugin will include system properties when writing the properties file. System properties override
+     * both environment variables and project properties.
+     * 
+     * @parameter default-value="false" property="properties.includeSystemProperties"
+     */
+    private boolean includeSystemProperties;
+
+    /**
+     * If true, the plugin will include environment variables when writing the properties file. Environment variables
+     * are prefixed with "env". Environment variables override project properties.
+     * 
+     * @parameter default-value="false" property="properties.includeEnvironmentVariables"
+     */
+    private boolean includeEnvironmentVariables;
+
+    /**
+     * Comma separated set of properties to exclude when writing the properties file
+     * 
+     * @parameter property="properties.exclude"
+     */
+    private String exclude;
+
+    /**
+     * Comma separated set of properties to write to the properties file. If provided, only the properties matching
+     * those supplied here will be written to the properties file.
+     * 
+     * @parameter property="properties.include"
+     */
+    private String include;
+
+    /* (non-Javadoc)
+     * @see org.apache.maven.plugin.AbstractMojo#execute()
+     */
+    @Override
+    @SuppressFBWarnings({"NP_UNWRITTEN_FIELD","UWF_UNWRITTEN_FIELD"})
+    public void execute() throws MojoExecutionException, MojoFailureException {
+        Properties properties = new Properties();
+        // Add project properties
+        properties.putAll(project.getProperties());
+        if (includeEnvironmentVariables) {
+            // Add environment variables, overriding any existing properties with the same key
+            properties.putAll(getEnvironmentVariables());
+        }
+        if (includeSystemProperties) {
+            // Add system properties, overriding any existing properties with the same key
+            properties.putAll(System.getProperties());
+        }
+
+        // Remove properties as appropriate
+        trim(properties, exclude, include);
+
+        String comment = "# " + new Date() + "\n";
+        List<String> escapeTokens = getEscapeChars(escapeChars);
+
+        getLog().info("Creating " + outputFile);
+        writeProperties(outputFile, comment, properties, escapeTokens);
+	    }
+
+    /**
+     * Provides environment variables.
+     * @return environment variables
+     */
+    protected static Properties getEnvironmentVariables() {
+        Properties props = new Properties();
+        for (Entry<String, String> entry : System.getenv().entrySet()) {
+            props.setProperty(PROPERTY_PREFIX_ENV + entry.getKey(), entry.getValue());
+        }
+        return props;
+    }
+
+    /**
+     * Removes properties which should not be written.
+     * @param properties
+     * @param omitCSV
+     * @param includeCSV
+     * @throws MojoExecutionException
+     */
+    protected void trim(Properties properties, String omitCSV, String includeCSV) throws MojoExecutionException {
+        List<String> omitKeys = getListFromCSV(omitCSV);
+        for (String key : omitKeys) {
+            properties.remove(key);
+        }
+        
+        List<String> includeKeys = getListFromCSV(includeCSV);
+//      mh: including keys from predefined properties
+        if (includePropertyKeysFromFiles!=null && includePropertyKeysFromFiles.length>0) {
+        	for (String currentIncludeLoc : includePropertyKeysFromFiles) {
+        		if (validate(currentIncludeLoc)) {
+        			Properties p = getProperties(currentIncludeLoc);
+        			for (String key : p.stringPropertyNames()) {
+        				includeKeys.add(key);
+        			}
+        		}
+        	}
+        }
+        if (includeKeys!=null && !includeKeys.isEmpty()) {
+//        	removing only when include keys provided
+        	Set<String> keys = properties.stringPropertyNames();
+            for (String key : keys) {
+                if (!includeKeys.contains(key)) {
+                    properties.remove(key);
+                }
+            }	
+        }
+    }
+
+    /**
+     * Checks whether file exists.
+     * @param location
+     * @return true when exists, false otherwise.
+     */
+    protected boolean exists(String location) {
+        if (StringUtils.isBlank(location)) {
+            return false;
+        }
+        File file = new File(location);
+        if (file.exists()) {
+            return true;
+        }
+        ResourceLoader loader = new DefaultResourceLoader();
+        Resource resource = loader.getResource(location);
+        return resource.exists();
+    }
+
+    /**
+     * Validates resource location.
+     * @param location
+     * @return true when valid, false otherwise
+     * @throws MojoExecutionException
+     */
+    protected boolean validate(String location) throws MojoExecutionException {
+        boolean exists = exists(location);
+        if (exists) {
+            return true;
+        }
+        if (quiet) {
+            getLog().info("Ignoring non-existent properties file '" + location + "'");
+            return false;
+        } else {
+            throw new MojoExecutionException("Non-existent properties file '" + location + "'");
+        }
+    }
+
+    /**
+     * Provides input stream.
+     * @param location
+     * @return input stream
+     * @throws IOException
+     */
+    protected InputStream getInputStream(String location) throws IOException {
+        File file = new File(location);
+        if (file.exists()) {
+            return new FileInputStream(location);
+        }
+        ResourceLoader loader = new DefaultResourceLoader();
+        Resource resource = loader.getResource(location);
+        return resource.getInputStream();
+    }
+
+    /**
+     * Creates properties for given location.
+     * @param location
+     * @return properties for given location
+     * @throws MojoExecutionException
+     */
+    protected Properties getProperties(String location) throws MojoExecutionException {
+        InputStream in = null;
+        try {
+            Properties properties = new Properties();
+            in = getInputStream(location);
+            if (location.toLowerCase().endsWith(".xml")) {
+                properties.loadFromXML(in);
+            } else {
+                properties.load(in);
+            }
+            return properties;
+        } catch (IOException e) {
+            throw new MojoExecutionException("Error reading properties file " + location, e);
+        } finally {
+            IOUtils.closeQuietly(in);
+        }
+    }
+
+    /**
+     * Provides escape characters.
+     * @param escapeChars
+     * @return escape characters
+     */
+    protected List<String> getEscapeChars(String escapeChars) {
+        List<String> tokens = getListFromCSV(escapeChars);
+        List<String> realTokens = new ArrayList<String>();
+        for (String token : tokens) {
+            String realToken = getRealToken(token);
+            realTokens.add(realToken);
+        }
+        return realTokens;
+    }
+
+    /**
+     * Provides real token.
+     * @param token
+     * @return real token
+     */
+    protected String getRealToken(String token) {
+        if (token.equalsIgnoreCase("CR")) {
+            return CR;
+        } else if (token.equalsIgnoreCase("LF")) {
+            return LF;
+        } else if (token.equalsIgnoreCase("TAB")) {
+            return TAB;
+        } else {
+            return token;
+        }
+    }
+
+    /**
+     * Returns content.
+     * @param comment
+     * @param properties
+     * @param escapeTokens
+     * @return content
+     */
+    protected String getContent(String comment, Properties properties, List<String> escapeTokens) {
+        List<String> names = new ArrayList<String>(properties.stringPropertyNames());
+        Collections.sort(names);
+        StringBuilder sb = new StringBuilder();
+        if (!StringUtils.isBlank(comment)) {
+            sb.append(comment);
+        }
+        for (String name : names) {
+            String value = properties.getProperty(name);
+            String escapedValue = escape(value, escapeTokens);
+            sb.append(name + "=" + escapedValue + "\n");
+        }
+        return sb.toString();
+    }
+
+    /**
+     * Writes properties to given file.
+     * @param file
+     * @param comment
+     * @param properties
+     * @param escapeTokens
+     * @throws MojoExecutionException
+     */
+    protected void writeProperties(File file, String comment, Properties properties, List<String> escapeTokens)
+            throws MojoExecutionException {
+        try {
+            String content = getContent(comment, properties, escapeTokens);
+            FileUtils.writeStringToFile(file, content, ENCODING_UTF8);
+        } catch (IOException e) {
+            throw new MojoExecutionException("Error creating properties file", e);
+        }
+    }
+	 
+    /**
+     * Escapes characters.
+     * @param s
+     * @param escapeChars
+     * @return
+     */
+    protected String escape(String s, List<String> escapeChars) {
+    	String result = s;
+        for (String escapeChar : escapeChars) {
+            result = result.replace(escapeChar, getReplacementToken(escapeChar));
+        }
+        return result;
+    }
+
+    /**
+     * Provides replacement token.
+     * @param escapeChar
+     * @return replacement token
+     */
+    protected String getReplacementToken(String escapeChar) {
+        if (escapeChar.equals(CR)) {
+            return "\\r";
+        } else if (escapeChar.equals(LF)) {
+            return "\\n";
+        } else if (escapeChar.equals(TAB)) {
+            return "\\t";
+        } else {
+            return "\\" + escapeChar;
+        }
+    }
+
+    /**
+	 * Returns list from csv.
+	 * @param csv
+	 * @return list of values generated from CSV
+	 */
+	protected static final List<String> getListFromCSV(String csv) {
+		if (StringUtils.isBlank(csv)) {
+			return new ArrayList<String>();
+		}
+		List<String> list = new ArrayList<String>();
+		String[] tokens = StringUtils.split(csv, ",");
+		for (String token : tokens) {
+			list.add(token.trim());
+		}
+		return list;
+	}
+
+    public void setIncludeSystemProperties(boolean includeSystemProperties) {
+        this.includeSystemProperties = includeSystemProperties;
+    }
+
+    public void setEscapeChars(String escapeChars) {
+        this.escapeChars = escapeChars;
+    }
+
+    public void setIncludeEnvironmentVariables(boolean includeEnvironmentVariables) {
+        this.includeEnvironmentVariables = includeEnvironmentVariables;
+    }
+
+    public void setExclude(String exclude) {
+        this.exclude = exclude;
+    }
+
+    public void setInclude(String include) {
+        this.include = include;
+    }
+
+    public void setQuiet(boolean quiet) {
+        this.quiet = quiet;
+    }
+	
+	/**
+	 * Sets property files for which keys properties should be included.
+	 * @param includePropertyKeysFromFiles
+	 */
+	public void setIncludePropertyKeysFromFiles(
+			String[] includePropertyKeysFromFiles) {
+		if (includePropertyKeysFromFiles!=null) {
+			this.includePropertyKeysFromFiles = Arrays.copyOf(
+					includePropertyKeysFromFiles, 
+					includePropertyKeysFromFiles.length);	
+		}
+	}
+	
+}
\ No newline at end of file
Index: dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin	(revision 51263)

Property changes on: dnet-hadoop/dhp-build/dhp-build-properties-maven-plugin
___________________________________________________________________
Added: svn:ignore
## -0,0 +1,7 ##
+*~
+*.iml
+.idea
+.classpath
+.project
+.settings
+target
Index: dnet-hadoop/dhp-schemas/dhp-schemas.iml
===================================================================
--- dnet-hadoop/dhp-schemas/dhp-schemas.iml	(nonexistent)
+++ dnet-hadoop/dhp-schemas/dhp-schemas.iml	(revision 51263)
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
+    <output url="file://$MODULE_DIR$/target/classes" />
+    <output-test url="file://$MODULE_DIR$/target/test-classes" />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
+      <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
+      <sourceFolder url="file://$MODULE_DIR$/src/main/avro" type="java-resource" />
+      <sourceFolder url="file://$MODULE_DIR$/target/generated-sources/avro" isTestSource="false" generated="true" />
+      <excludeFolder url="file://$MODULE_DIR$/target" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="Maven: org.apache.avro:avro:1.7.6-cdh5.9.0" level="project" />
+    <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-core-asl:1.8.8" level="project" />
+    <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-mapper-asl:1.8.8" level="project" />
+    <orderEntry type="library" name="Maven: com.thoughtworks.paranamer:paranamer:2.8" level="project" />
+    <orderEntry type="library" name="Maven: org.xerial.snappy:snappy-java:1.0.4.1" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.4.1" level="project" />
+    <orderEntry type="library" name="Maven: org.tukaani:xz:1.0" level="project" />
+    <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.22" level="project" />
+    <orderEntry type="library" name="Maven: org.slf4j:slf4j-log4j12:1.7.22" level="project" />
+    <orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="Maven: javax.servlet:javax.servlet-api:3.1.0" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.12" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-all:1.3" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.mockito:mockito-all:1.10.19" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.powermock:powermock-core:1.6.6" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.powermock:powermock-reflect:1.6.6" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.objenesis:objenesis:2.4" level="project" />
+    <orderEntry type="library" scope="TEST" name="Maven: org.javassist:javassist:3.21.0-GA" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.code.findbugs:annotations:3.0.1" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: net.jcip:jcip-annotations:1.0" level="project" />
+    <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.code.findbugs:jsr305:3.0.1" level="project" />
+  </component>
+</module>
\ No newline at end of file
Index: dnet-hadoop/dhp-build/pom.xml
===================================================================
--- dnet-hadoop/dhp-build/pom.xml	(nonexistent)
+++ dnet-hadoop/dhp-build/pom.xml	(revision 51263)
@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>eu.dnetlib.dhp</groupId>
+		<artifactId>dhp</artifactId>
+		<version>1.0.0-SNAPSHOT</version>
+	</parent>
+	<artifactId>dhp-build</artifactId>
+	<packaging>pom</packaging>
+	<modules>
+		<module>dhp-build-assembly-resources</module>
+		<module>dhp-build-properties-maven-plugin</module>
+	</modules>
+
+</project>
Index: dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/upload_workflow.sh
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/upload_workflow.sh	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/commands/upload_workflow.sh	(revision 51263)
@@ -0,0 +1,34 @@
+#!/bin/bash
+exec 3>&1
+BASH_XTRACEFD=3
+set -x ## print every executed command
+
+
+if [ $# = 0 ] ; then
+    target_dir_root=`pwd`'/${oozieAppDir}'
+else
+    target_dir_root=`readlink -f $1`'/${oozieAppDir}'
+fi
+
+# initial phase, creating symbolic links to jars in all subworkflows
+# currently disabled
+#libDir=$target_dir_root'/lib'
+#dirs=`find $target_dir_root/* -maxdepth 10 -type d`
+#for dir in $dirs
+#do
+#        if [ -f $dir/workflow.xml ]
+#        then
+#                echo "creating symbolic links to jars in directory: $dir/lib"
+#                if [ ! -d "$dir/lib" ]; then
+#                        mkdir $dir/lib
+#                fi
+#                find $libDir -type f -exec ln -s \{\} $dir/lib \;
+#        fi
+#done
+
+
+#uploading
+hadoop fs -rm -r ${sandboxDir}
+hadoop fs -mkdir -p ${sandboxDir}
+hadoop fs -mkdir -p ${workingDir}
+hadoop fs -put $target_dir_root ${sandboxDir}
Index: dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/assemblies/tests.xml
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/assemblies/tests.xml	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-assembly-resources/src/main/resources/assemblies/tests.xml	(revision 51263)
@@ -0,0 +1,24 @@
+<assembly
+	xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+
+	<id>tests</id>
+	<formats>
+		<format>jar</format>
+	</formats>
+	<includeBaseDirectory>false</includeBaseDirectory>
+	<fileSets>
+		<fileSet>
+			<directory>${project.build.testOutputDirectory}
+                </directory>
+			<outputDirectory />
+		</fileSet>
+	</fileSets>
+<!--  <dependencySets>
+		<dependencySet>
+			<useProjectArtifact>false</useProjectArtifact>
+			<outputDirectory>lib</outputDirectory>
+		</dependencySet>
+	</dependencySets>-->
+</assembly>
\ No newline at end of file
Index: dnet-hadoop/dhp-build/dhp-build-assembly-resources
===================================================================
--- dnet-hadoop/dhp-build/dhp-build-assembly-resources	(nonexistent)
+++ dnet-hadoop/dhp-build/dhp-build-assembly-resources	(revision 51263)

Property changes on: dnet-hadoop/dhp-build/dhp-build-assembly-resources
___________________________________________________________________
Added: svn:ignore
## -0,0 +1,7 ##
+*~
+*.iml
+.idea
+.classpath
+.project
+.settings
+target
Index: dnet-hadoop/dhp-build
===================================================================
--- dnet-hadoop/dhp-build	(nonexistent)
+++ dnet-hadoop/dhp-build	(revision 51263)

Property changes on: dnet-hadoop/dhp-build
___________________________________________________________________
Added: svn:ignore
## -0,0 +1,6 ##
+*.iml
+.idea
+.classpath
+.project
+.settings
+target
Index: dnet-hadoop/pom.xml
===================================================================
--- dnet-hadoop/pom.xml	(nonexistent)
+++ dnet-hadoop/pom.xml	(revision 51263)
@@ -0,0 +1,950 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+	<modelVersion>4.0.0</modelVersion>
+
+	<groupId>eu.dnetlib.dhp</groupId>
+	<artifactId>dhp</artifactId>
+	<version>1.0.0-SNAPSHOT</version>
+	<packaging>pom</packaging>
+
+	<url>http://www.d-net.research-infrastructures.eu</url>
+
+	<licenses>
+		<license>
+			<name>The Apache Software License, Version 2.0</name>
+			<url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+			<distribution>repo</distribution>
+			<comments>A business-friendly OSS license</comments>
+		</license>
+	</licenses>
+
+	<modules>
+		<module>dhp-build</module>
+		<module>dhp-common</module>
+		<module>dhp-schemas</module>
+		<module>dhp-wf</module>
+	</modules>
+
+	<issueManagement>
+		<system>Redmine</system>
+		<url>https://issue.openaire.research-infrastructures.eu/projects/openaire</url>
+	</issueManagement>
+
+	<ciManagement>
+		<system>jenkins</system>
+		<url>https://jenkins-dnet.d4science.org/</url>
+	</ciManagement>
+
+	<scm>
+		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/dnet-hadoop</developerConnection>
+	</scm>
+
+	<pluginRepositories>
+	</pluginRepositories>
+
+	<repositories>
+
+		<repository>
+			<id>dnet45-releases</id>
+			<name>D-Net 45 releases</name>
+			<url>http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-releases</url>
+			<layout>default</layout>
+			<snapshots>
+				<enabled>false</enabled>
+			</snapshots>
+			<releases>
+				<enabled>true</enabled>
+			</releases>
+		</repository>
+		<repository>
+			<id>dnet45-bootstrap-release</id>
+			<name>dnet45 bootstrap release</name>
+			<url>http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-bootstrap-release</url>
+			<layout>default</layout>
+			<snapshots>
+				<enabled>false</enabled>
+			</snapshots>
+			<releases>
+				<enabled>true</enabled>
+			</releases>
+		</repository>
+
+		<repository>
+			<id>cloudera</id>
+			<name>Cloudera Repository</name>
+			<url>https://repository.cloudera.com/artifactory/cloudera-repos</url>
+			<releases>
+				<enabled>true</enabled>
+			</releases>
+			<snapshots>
+				<enabled>false</enabled>
+			</snapshots>
+		</repository>
+
+	</repositories>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-api</artifactId>
+			<version>1.7.22</version>
+		</dependency>
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-log4j12</artifactId>
+			<version>1.7.22</version>
+		</dependency>
+		<dependency>
+			<groupId>log4j</groupId>
+			<artifactId>log4j</artifactId>
+			<version>1.2.17</version>
+		</dependency>
+		<dependency>
+			<groupId>javax.servlet</groupId>
+			<artifactId>javax.servlet-api</artifactId>
+			<version>3.1.0</version>
+			<scope>runtime</scope>
+		</dependency>
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<version>4.12</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.hamcrest</groupId>
+			<artifactId>hamcrest-all</artifactId>
+			<version>1.3</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.mockito</groupId>
+			<artifactId>mockito-all</artifactId>
+			<version>1.10.19</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.powermock</groupId>
+			<artifactId>powermock-core</artifactId>
+			<version>1.6.6</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>com.google.code.findbugs</groupId>
+			<artifactId>annotations</artifactId>
+			<version>3.0.1</version>
+			<scope>provided</scope>
+		</dependency>
+		<dependency>
+			<groupId>com.google.code.findbugs</groupId>
+			<artifactId>jsr305</artifactId>
+			<version>3.0.1</version>
+			<scope>provided</scope>
+		</dependency>
+	</dependencies>
+
+	<dependencyManagement>
+		<dependencies>
+
+			<dependency>
+				<groupId>org.apache.hadoop</groupId>
+				<artifactId>hadoop-common</artifactId>
+				<version>${dhp.hadoop.version}</version>
+				<scope>provided</scope>
+				<exclusions>
+					<exclusion>
+						<artifactId>servlet-api</artifactId>
+						<groupId>javax.servlet</groupId>
+					</exclusion>
+				</exclusions>
+			</dependency>
+
+			<dependency>
+				<groupId>org.apache.hadoop</groupId>
+				<artifactId>hadoop-yarn-common</artifactId>
+				<version>${dhp.hadoop.version}</version>
+				<scope>provided</scope>
+			</dependency>
+			<dependency>
+				<groupId>org.apache.hadoop</groupId>
+				<artifactId>hadoop-mapreduce-client-common</artifactId>
+				<version>${dhp.hadoop.version}</version>
+				<scope>provided</scope>
+			</dependency>
+
+			<dependency>
+				<groupId>org.apache.hadoop</groupId>
+				<artifactId>hadoop-yarn-common</artifactId>
+				<version>${dhp.hadoop.version}</version>
+				<scope>test</scope>
+				<type>test-jar</type>
+			</dependency>
+
+			<dependency>
+				<groupId>org.apache.hadoop</groupId>
+				<artifactId>hadoop-mapreduce-client-common</artifactId>
+				<version>${dhp.hadoop.version}</version>
+				<scope>test</scope>
+				<type>test-jar</type>
+			</dependency>
+
+			<dependency>
+				<groupId>org.apache.hadoop</groupId>
+				<artifactId>hadoop-mapreduce-client-app</artifactId>
+				<version>${dhp.hadoop.version}</version>
+				<scope>provided</scope>
+				<exclusions>
+					<exclusion>
+						<artifactId>servlet-api</artifactId>
+						<groupId>javax.servlet</groupId>
+					</exclusion>
+				</exclusions>
+			</dependency>
+
+			<dependency>
+				<groupId>org.apache.hadoop</groupId>
+				<artifactId>hadoop-mapreduce-client-core</artifactId>
+				<version>${dhp.hadoop.version}</version>
+				<scope>provided</scope>
+				<exclusions>
+					<exclusion>
+						<artifactId>servlet-api</artifactId>
+						<groupId>javax.servlet</groupId>
+					</exclusion>
+					<!-- added temporarily due to unresolvable transitive dependencies: more in #2810#note-3 -->
+					<exclusion>
+						<groupId>org.apache.calcite</groupId>
+						<artifactId>calcite-core</artifactId>
+					</exclusion>
+					<exclusion>
+						<groupId>org.apache.calcite</groupId>
+						<artifactId>calcite-avatica</artifactId>
+					</exclusion>
+				</exclusions>
+			</dependency>
+
+			<dependency>
+				<groupId>org.apache.hadoop</groupId>
+				<artifactId>hadoop-hdfs</artifactId>
+				<version>${dhp.hadoop.version}</version>
+			</dependency>
+
+			<dependency>
+				<groupId>org.apache.hadoop</groupId>
+				<artifactId>hadoop-hdfs</artifactId>
+				<version>${dhp.hadoop.version}</version>
+				<type>test-jar</type>
+				<scope>test</scope>
+			</dependency>
+
+
+			<!-- Oozie dependencies -->
+			<dependency>
+				<groupId>org.apache.oozie</groupId>
+				<artifactId>oozie-core</artifactId>
+				<version>${dhp.oozie.version}</version>
+				<exclusions>
+					<!-- conflicts -->
+					<exclusion>
+						<artifactId>servlet-api</artifactId>
+						<groupId>javax.servlet</groupId>
+					</exclusion>
+					<!-- added due to unresolvable transitive dependencies: more in #787 -->
+					<exclusion>
+						<groupId>org.apache.calcite</groupId>
+						<artifactId>calcite-core</artifactId>
+					</exclusion>
+					<exclusion>
+						<groupId>org.apache.calcite</groupId>
+						<artifactId>calcite-avatica</artifactId>
+					</exclusion>
+				</exclusions>
+			</dependency>
+
+			<dependency>
+				<groupId>org.apache.oozie</groupId>
+				<artifactId>oozie-client</artifactId>
+				<version>${dhp.oozie.version}</version>
+				<exclusions>
+					<!-- conflicts -->
+					<exclusion>
+						<artifactId>slf4j-simple</artifactId>
+						<groupId>org.slf4j</groupId>
+					</exclusion>
+				</exclusions>
+			</dependency>
+
+			<dependency>
+				<groupId>org.apache.spark</groupId>
+				<artifactId>spark-core_2.10</artifactId>
+				<version>${dhp.spark.version}</version>
+				<scope>provided</scope>
+			</dependency>
+
+			<dependency>
+				<groupId>org.apache.spark</groupId>
+				<artifactId>spark-sql_2.10</artifactId>
+				<version>${dhp.spark.version}</version>
+				<scope>provided</scope>
+			</dependency>
+
+			<dependency>
+				<groupId>com.databricks</groupId>
+				<artifactId>spark-avro_2.10</artifactId>
+				<version>1.1.0-${dhp.cdh.version}</version>
+			</dependency>
+
+			<dependency>
+				<groupId>com.databricks</groupId>
+				<artifactId>spark-csv_2.10</artifactId>
+				<version>1.5.0</version>
+			</dependency>
+
+			<dependency>
+				<groupId>pl.edu.icm.spark-utils</groupId>
+				<artifactId>spark-utils</artifactId>
+				<version>1.0.0</version>
+			</dependency>
+
+			<dependency>
+				<groupId>org.mongodb.spark</groupId>
+				<artifactId>mongo-spark-connector_2.10</artifactId>
+				<version>2.2.1</version>
+				<scope>provided</scope>
+			</dependency>
+
+
+			<!-- Avro dependencies -->
+			<dependency>
+				<groupId>org.apache.avro</groupId>
+				<artifactId>avro</artifactId>
+				<version>${dhp.avro.version}</version>
+			</dependency>
+
+			<dependency>
+				<groupId>org.apache.avro</groupId>
+				<artifactId>avro-mapred</artifactId>
+				<version>${dhp.avro.version}</version>
+				<classifier>hadoop2</classifier>
+				<exclusions>
+					<!-- conflicts -->
+					<exclusion>
+						<artifactId>servlet-api</artifactId>
+						<groupId>org.mortbay.jetty</groupId>
+					</exclusion>
+					<exclusion>
+						<artifactId>netty</artifactId>
+						<groupId>io.netty</groupId>
+					</exclusion>
+				</exclusions>
+			</dependency>
+
+
+			<!-- Pig dependencies -->
+			<dependency>
+				<groupId>org.apache.pig</groupId>
+				<artifactId>pig</artifactId>
+				<version>${dhp.pig.version}</version>
+				<scope>provided</scope>
+				<exclusions>
+					<exclusion>
+						<groupId>org.mortbay.jetty</groupId>
+						<artifactId>servlet-api-2.5</artifactId>
+					</exclusion>
+					<exclusion>
+						<artifactId>servlet-api</artifactId>
+						<groupId>javax.servlet</groupId>
+					</exclusion>
+				</exclusions>
+			</dependency>
+
+			<dependency>
+				<groupId>org.apache.pig</groupId>
+				<artifactId>piggybank</artifactId>
+				<version>${dhp.pig.version}</version>
+				<scope>provided</scope>
+			</dependency>
+
+			<dependency>
+				<groupId>org.apache.pig</groupId>
+				<artifactId>pigunit</artifactId>
+				<version>${dhp.pig.version}</version>
+			</dependency>
+
+			<!-- Cermine dependencies -->
+			<dependency>
+				<groupId>pl.edu.icm.cermine</groupId>
+				<artifactId>cermine-impl</artifactId>
+				<version>1.13</version>
+			</dependency>
+
+			<!-- Coansys dependencies -->
+			<dependency>
+				<groupId>pl.edu.icm.coansys</groupId>
+				<artifactId>models</artifactId>
+				<version>${dhp.coansys.version}</version>
+			</dependency>
+
+			<dependency>
+				<groupId>pl.edu.icm.coansys</groupId>
+				<artifactId>citation-matching-core-code</artifactId>
+				<version>${dhp.coansys.version}</version>
+				<exclusions>
+					<!-- unneded cdh5 which should be provided-->
+					<exclusion>
+						<groupId>org.apache.avro</groupId>
+						<artifactId>avro-mapred</artifactId>
+					</exclusion>
+					<exclusion>
+						<groupId>org.apache.hadoop</groupId>
+						<artifactId>hadoop-client</artifactId>
+					</exclusion>
+					<exclusion>
+						<groupId>org.apache.hadoop</groupId>
+						<artifactId>hadoop-yarn-api</artifactId>
+					</exclusion>
+					<exclusion>
+						<groupId>org.apache.hadoop</groupId>
+						<artifactId>hadoop-yarn-client</artifactId>
+					</exclusion>
+					<exclusion>
+						<groupId>org.apache.hadoop</groupId>
+						<artifactId>hadoop-mapreduce-client-common</artifactId>
+					</exclusion>
+					<exclusion>
+						<groupId>org.apache.hadoop</groupId>
+						<artifactId>hadoop-mapreduce-client-app</artifactId>
+					</exclusion>
+					<exclusion>
+						<groupId>org.apache.hadoop</groupId>
+						<artifactId>hadoop-mapreduce-client-core</artifactId>
+					</exclusion>
+					<exclusion>
+						<!-- conflicts -->
+						<groupId>org.slf4j</groupId>
+						<artifactId>slf4j-simple</artifactId>
+					</exclusion>
+				</exclusions>
+
+			</dependency>
+
+			<!-- dnet dependencies -->
+			<dependency>
+				<groupId>eu.dnetlib</groupId>
+				<artifactId>dnet-openaireplus-mapping-utils</artifactId>
+				<version>[6.0.0, 7.0.0)</version>
+				<exclusions>
+					<!-- unneeded dependencies -->
+					<exclusion>
+						<groupId>org.apache.hadoop</groupId>
+						<artifactId>hadoop-core</artifactId>
+					</exclusion>
+					<exclusion>
+						<groupId>org.apache.hadoop</groupId>
+						<artifactId>hadoop-hdfs</artifactId>
+					</exclusion>
+					<exclusion>
+						<groupId>org.apache.zookeeper</groupId>
+						<artifactId>zookeeper</artifactId>
+					</exclusion>
+					<exclusion>
+						<groupId>jgrapht</groupId>
+						<artifactId>jgrapht</artifactId>
+					</exclusion>
+				</exclusions>
+			</dependency>
+
+			<dependency>
+				<groupId>eu.dnetlib</groupId>
+				<artifactId>dnet-objectstore-rmi</artifactId>
+				<version>[2.0.0, 3.0.0)</version>
+			</dependency>
+
+			<dependency>
+				<groupId>eu.dnetlib</groupId>
+				<artifactId>cnr-rmi-api</artifactId>
+				<version>[2.0.0, 3.0.0)</version>
+			</dependency>
+
+			<dependency>
+				<groupId>eu.dnetlib</groupId>
+				<artifactId>cnr-resultset-client</artifactId>
+				<version>[2.0.0, 3.0.0)</version>
+				<exclusions>
+					<!-- unneeded -->
+					<exclusion>
+						<groupId>org.springframework</groupId>
+						<artifactId>spring-web</artifactId>
+					</exclusion>
+					<exclusion>
+						<groupId>org.springframework</groupId>
+						<artifactId>spring-webmvc</artifactId>
+					</exclusion>
+				</exclusions>
+			</dependency>
+
+			<dependency>
+				<groupId>eu.dnetlib</groupId>
+				<artifactId>dnet-actionmanager-common</artifactId>
+				<version>[6.0.0, 7.0.0)</version>
+				<exclusions>
+					<!-- duplicate with different groupId -->
+					<exclusion>
+						<groupId>apache</groupId>
+						<artifactId>commons-logging</artifactId>
+					</exclusion>
+				</exclusions>
+			</dependency>
+
+			<dependency>
+				<groupId>eu.dnetlib</groupId>
+				<artifactId>cnr-service-utils</artifactId>
+				<version>[1.0.0, 2.0.0)</version>
+			</dependency>
+
+			<!-- command line arguments parser -->
+			<dependency>
+				<groupId>com.beust</groupId>
+				<artifactId>jcommander</artifactId>
+				<version>1.60</version>
+			</dependency>
+
+			<dependency>
+				<groupId>com.google.code.gson</groupId>
+				<artifactId>gson</artifactId>
+				<version>2.8.0</version>
+			</dependency>
+
+			<dependency>
+				<groupId>com.googlecode.json-simple</groupId>
+				<artifactId>json-simple</artifactId>
+				<version>1.1.1</version>
+			</dependency>
+
+			<dependency>
+				<groupId>org.apache.commons</groupId>
+				<artifactId>commons-lang3</artifactId>
+				<version>3.5</version>
+			</dependency>
+
+			<dependency>
+				<groupId>org.apache.commons</groupId>
+				<artifactId>commons-collections4</artifactId>
+				<version>4.1</version>
+			</dependency>
+
+			<dependency>
+				<groupId>com.thoughtworks.xstream</groupId>
+				<artifactId>xstream</artifactId>
+				<version>1.4.9</version>
+			</dependency>
+
+			<dependency>
+				<groupId>xalan</groupId>
+				<artifactId>xalan</artifactId>
+				<version>2.7.2</version>
+			</dependency>
+
+			<dependency>
+				<groupId>xml-apis</groupId>
+				<artifactId>xml-apis</artifactId>
+				<version>1.4.01</version>
+			</dependency>
+
+			<dependency>
+				<groupId>org.jdom</groupId>
+				<artifactId>jdom</artifactId>
+				<version>1.1.3</version>
+			</dependency>
+
+			<dependency>
+				<groupId>org.jsoup</groupId>
+				<artifactId>jsoup</artifactId>
+				<version>1.10.2</version>
+			</dependency>
+
+			<dependency>
+				<groupId>net.sf.opencsv</groupId>
+				<artifactId>opencsv</artifactId>
+				<version>2.3</version>
+			</dependency>
+
+			<dependency>
+				<groupId>com.googlecode.protobuf-java-format</groupId>
+				<artifactId>protobuf-java-format</artifactId>
+				<version>1.2</version>
+			</dependency>
+
+			<dependency>
+				<groupId>com.google.protobuf</groupId>
+				<artifactId>protobuf-java</artifactId>
+				<version>2.5.0</version>
+			</dependency>
+
+			<!-- this version is required by hadoop-core 2.6.0 -->
+			<dependency>
+				<groupId>com.google.guava</groupId>
+				<artifactId>guava</artifactId>
+				<version>12.0</version>
+			</dependency>
+
+			<dependency>
+				<groupId>commons-cli</groupId>
+				<artifactId>commons-cli</artifactId>
+				<version>1.3.1</version>
+			</dependency>
+
+			<dependency>
+				<groupId>commons-io</groupId>
+				<artifactId>commons-io</artifactId>
+				<version>2.5</version>
+			</dependency>
+
+			<dependency>
+				<groupId>de.sven-jacobs</groupId>
+				<artifactId>loremipsum</artifactId>
+				<version>1.0</version>
+			</dependency>
+
+			<dependency>
+				<groupId>net.schmizz</groupId>
+				<artifactId>sshj</artifactId>
+				<version>0.10.0</version>
+			</dependency>
+
+			<!-- this version has to be aligned with net.schmizz:sshj -->
+			<dependency>
+				<groupId>org.bouncycastle</groupId>
+				<artifactId>bcprov-jdk15on</artifactId>
+				<version>1.50</version>
+			</dependency>
+
+			<!-- Needed to run Avro-based map-reduce -->
+			<dependency>
+				<groupId>com.thoughtworks.paranamer</groupId>
+				<artifactId>paranamer</artifactId>
+				<version>2.8</version>
+			</dependency>
+
+			<!-- Needed to run Pig jobs -->
+			<dependency>
+				<groupId>com.linkedin.datafu</groupId>
+				<artifactId>datafu</artifactId>
+				<version>1.2.0</version>
+			</dependency>
+
+			<dependency>
+				<groupId>org.apache.cxf</groupId>
+				<artifactId>cxf-rt-frontend-jaxws</artifactId>
+				<version>${cxf.version}</version>
+			</dependency>
+
+			<!-- this version has to be aligned with cdh5.9.0 sharelib -->
+			<dependency>
+				<groupId>com.sun.xml.bind</groupId>
+				<artifactId>jaxb-impl</artifactId>
+				<version>2.2.7</version>
+				<scope>runtime</scope>
+			</dependency>
+
+			<dependency>
+				<groupId>org.springframework</groupId>
+				<artifactId>spring-beans</artifactId>
+				<version>${spring.version}</version>
+			</dependency>
+
+			<dependency>
+				<groupId>org.springframework</groupId>
+				<artifactId>spring-context</artifactId>
+				<version>${spring.version}</version>
+			</dependency>
+
+			<dependency>
+				<groupId>org.scala-lang</groupId>
+				<artifactId>scala-library</artifactId>
+				<version>${scala.version}</version>
+			</dependency>
+
+			<dependency>
+				<groupId>commons-beanutils</groupId>
+				<artifactId>commons-beanutils</artifactId>
+				<version>1.9.3</version>
+			</dependency>
+
+			<dependency>
+				<groupId>org.apache.curator</groupId>
+				<artifactId>curator-test</artifactId>
+				<version>3.3.0</version>
+				<scope>test</scope>
+			</dependency>
+
+		</dependencies>
+	</dependencyManagement>
+
+
+	<build>
+		<directory>target</directory>
+		<outputDirectory>target/classes</outputDirectory>
+		<finalName>${project.artifactId}-${project.version}</finalName>
+		<testOutputDirectory>target/test-classes</testOutputDirectory>
+
+		<pluginManagement>
+			<plugins>
+
+				<plugin>
+					<groupId>org.apache.maven.plugins</groupId>
+					<artifactId>maven-compiler-plugin</artifactId>
+					<version>3.6.0</version>
+					<configuration>
+						<source>1.8</source>
+						<target>1.8</target>
+						<encoding>${project.build.sourceEncoding}</encoding>
+					</configuration>
+				</plugin>
+
+				<plugin>
+					<groupId>org.apache.maven.plugins</groupId>
+					<artifactId>maven-jar-plugin</artifactId>
+					<version>3.0.2</version>
+				</plugin>
+
+				<plugin>
+					<groupId>org.apache.maven.plugins</groupId>
+					<artifactId>maven-source-plugin</artifactId>
+					<version>3.0.1</version>
+					<executions>
+						<execution>
+							<id>attach-sources</id>
+							<phase>verify</phase>
+							<goals>
+								<goal>jar-no-fork</goal>
+							</goals>
+						</execution>
+					</executions>
+				</plugin>
+
+				<plugin>
+					<groupId>org.apache.maven.plugins</groupId>
+					<artifactId>maven-surefire-plugin</artifactId>
+					<version>2.19.1</version>
+					<configuration>
+						<redirectTestOutputToFile>true</redirectTestOutputToFile>
+					</configuration>
+				</plugin>
+
+				<plugin>
+					<groupId>org.apache.maven.plugins</groupId>
+					<artifactId>maven-javadoc-plugin</artifactId>
+					<version>2.10.4</version>
+					<configuration>
+						<detectLinks>true</detectLinks>
+					</configuration>
+				</plugin>
+
+				<plugin>
+					<groupId>org.apache.maven.plugins</groupId>
+					<artifactId>maven-dependency-plugin</artifactId>
+					<version>3.0.0</version>
+				</plugin>
+
+				<plugin>
+					<groupId>net.alchim31.maven</groupId>
+					<artifactId>scala-maven-plugin</artifactId>
+					<version>3.2.2</version>
+					<executions>
+						<execution>
+							<id>scala-compile-first</id>
+							<phase>process-resources</phase>
+							<goals>
+								<goal>add-source</goal>
+								<goal>compile</goal>
+							</goals>
+						</execution>
+						<execution>
+							<id>scala-test-compile</id>
+							<phase>process-test-resources</phase>
+							<goals>
+								<goal>testCompile</goal>
+							</goals>
+						</execution>
+					</executions>
+				</plugin>
+
+				<!-- Plugin that generates Java classes from Avro schemas -->
+				<plugin>
+					<groupId>org.apache.avro</groupId>
+					<artifactId>avro-maven-plugin</artifactId>
+					<version>1.7.7</version>
+				</plugin>
+
+				<plugin>
+					<groupId>org.codehaus.mojo</groupId>
+					<artifactId>build-helper-maven-plugin</artifactId>
+					<version>1.12</version>
+				</plugin>
+
+				<!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself. -->
+				<plugin>
+					<groupId>org.eclipse.m2e</groupId>
+					<artifactId>lifecycle-mapping</artifactId>
+					<version>1.0.0</version>
+					<configuration>
+						<lifecycleMappingMetadata>
+							<pluginExecutions>
+								<pluginExecution>
+									<pluginExecutionFilter>
+										<groupId>
+											org.apache.avro
+										</groupId>
+										<artifactId>
+											avro-maven-plugin
+										</artifactId>
+										<versionRange>
+											[1.7.4,)
+										</versionRange>
+										<goals>
+											<goal>idl-protocol</goal>
+											<goal>schema</goal>
+										</goals>
+									</pluginExecutionFilter>
+									<action>
+										<ignore/>
+									</action>
+								</pluginExecution>
+								<pluginExecution>
+									<pluginExecutionFilter>
+										<groupId>
+											org.codehaus.mojo
+										</groupId>
+										<artifactId>
+											build-helper-maven-plugin
+										</artifactId>
+										<versionRange>
+											[1.7,)
+										</versionRange>
+										<goals>
+											<goal>add-source</goal>
+										</goals>
+									</pluginExecutionFilter>
+									<action>
+										<ignore/>
+									</action>
+								</pluginExecution>
+								<pluginExecution>
+									<pluginExecutionFilter>
+										<groupId>
+											org.apache.maven.plugins
+										</groupId>
+										<artifactId>
+											maven-plugin-plugin
+										</artifactId>
+										<versionRange>
+											[3.2,)
+										</versionRange>
+										<goals>
+											<goal>descriptor</goal>
+										</goals>
+									</pluginExecutionFilter>
+									<action>
+										<ignore></ignore>
+									</action>
+								</pluginExecution>
+							</pluginExecutions>
+						</lifecycleMappingMetadata>
+					</configuration>
+				</plugin>
+			</plugins>
+		</pluginManagement>
+
+		<plugins>
+
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-release-plugin</artifactId>
+				<version>2.5.3</version>
+			</plugin>
+
+			<plugin>
+				<groupId>org.jacoco</groupId>
+				<artifactId>jacoco-maven-plugin</artifactId>
+				<version>0.7.9</version>
+				<configuration>
+					<excludes>
+						<exclude>**/schemas/*</exclude>
+						<exclude>**/com/cloudera/**/*</exclude>
+						<exclude>**/org/apache/avro/io/**/*</exclude>
+					</excludes>
+				</configuration>
+				<executions>
+					<execution>
+						<id>default-prepare-agent</id>
+						<goals>
+							<goal>prepare-agent</goal>
+						</goals>
+					</execution>
+					<execution>
+						<id>default-report</id>
+						<phase>prepare-package</phase>
+						<goals>
+							<goal>report</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+
+		<extensions>
+			<extension>
+				<groupId>org.apache.maven.wagon</groupId>
+				<artifactId>wagon-ssh</artifactId>
+				<version>2.10</version>
+			</extension>
+		</extensions>
+
+	</build>
+
+
+	<distributionManagement>
+		<snapshotRepository>
+			<id>dnet45-snapshots</id>
+			<name>DNet45 Snapshots</name>
+			<url>http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots</url>
+			<layout>default</layout>
+		</snapshotRepository>
+		<repository>
+			<id>dnet45-releases</id>
+			<url>http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-releases</url>
+		</repository>
+	</distributionManagement>
+
+	<reporting>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-javadoc-plugin</artifactId>
+				<version>2.10.4</version>
+				<configuration>
+					<detectLinks>true</detectLinks>
+				</configuration>
+			</plugin>
+		</plugins>
+	</reporting>
+
+	<properties>
+		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+		<spring.version>[4.2.5.RELEASE]</spring.version>
+		<cxf.version>[3.1.5]</cxf.version>
+
+		<dhp.cdh.version>cdh5.9.0</dhp.cdh.version>
+
+		<dhp.oozie.version>4.1.0-${dhp.cdh.version}</dhp.oozie.version>
+		<dhp.pig.version>0.12.0-${dhp.cdh.version}</dhp.pig.version>
+		<dhp.avro.version>1.7.6-${dhp.cdh.version}</dhp.avro.version>
+		<dhp.hadoop.version>2.6.0-${dhp.cdh.version}</dhp.hadoop.version>
+		<dhp.spark.version>1.6.0-${dhp.cdh.version}</dhp.spark.version>
+		<scala.version>2.10.6</scala.version>
+	</properties>
+</project>
+
Index: dnet-hadoop/dhp-schemas/src/main/avro/eu/dnetlib/dhp/common/ReportEntry.avdl
===================================================================
--- dnet-hadoop/dhp-schemas/src/main/avro/eu/dnetlib/dhp/common/ReportEntry.avdl	(nonexistent)
+++ dnet-hadoop/dhp-schemas/src/main/avro/eu/dnetlib/dhp/common/ReportEntry.avdl	(revision 51263)
@@ -0,0 +1,16 @@
+@namespace("eu.dnetlib.dhp.common.schemas")
+protocol DHP{
+
+    enum ReportEntryType {
+        COUNTER, DURATION
+    }
+    
+    
+    record ReportEntry {
+    
+        string key;
+        ReportEntryType type;
+        string value;
+    
+    }
+}
Index: dnet-hadoop/dhp-schemas
===================================================================
--- dnet-hadoop/dhp-schemas	(nonexistent)
+++ dnet-hadoop/dhp-schemas	(revision 51263)

Property changes on: dnet-hadoop/dhp-schemas
___________________________________________________________________
Added: svn:ignore
## -0,0 +1,7 ##
+*~
+*.iml
+.idea
+.classpath
+.project
+.settings
+target
Index: dnet-hadoop/README.md
===================================================================
--- dnet-hadoop/README.md	(nonexistent)
+++ dnet-hadoop/README.md	(revision 51263)
@@ -0,0 +1 @@
+# dnet-hadoop
Index: dnet-hadoop
===================================================================
--- dnet-hadoop	(revision 51262)
+++ dnet-hadoop	(revision 51263)

Property changes on: dnet-hadoop
___________________________________________________________________
Added: svn:ignore
## -0,0 +1,7 ##
+.git*
+*.iml
+.idea
+.classpath
+.project
+.settings
+target
