D-Net

#!/bin/bash

## Generate example workflow app for given user and host name

HADOOP_USER=$1

HADOOP_HOST=$2

WORKFLOW_NAME=citation_metrics_advanced_workflow_app

WORKFLOW_PATH=src/main/resources/eu/dnetlib/iis/citation-metrics-advanced

mvn clean

echo "generating package \"$WORKFLOW_NAME\" in directory \"target/$WORKFLOW_NAME\""

mvn package -Poozie -Doozie.package.file.name=$WORKFLOW_NAME -Dworkflow.source.dir=$WORKFLOW_PATH -DsandboxDir=/user/$HADOOP_USER/iis_sandbox/$WORKFLOW_NAME -Duser.name=$HADOOP_USER -Dhost.name=$HADOOP_HOST

#!/bin/bash

## Generate a sample workflow app to be run on OpenAIRE+ hadoop cluster by user named "mateusz.kobos"

SCRIPTS_DIR=$(dirname $0)

HADOOP_USER=lars.holmnielsen

$SCRIPTS_DIR/generate_example_workflow_app-generic.sh $HADOOP_USER hadoop-master.openaire.eu

#!/bin/bash

## Generate a sample workflow app to be run on local Hadoop instance by the current user

SCRIPTS_DIR=$(dirname $0)

$SCRIPTS_DIR/generate_example_workflow_app-generic.sh $USER localhost

ssh lars.holmnielsen@hadoop.openaire.eu "rm -Rf /home/lars.holmnielsen/tmp/*.*"

scp target citation_metrics_advanced_workflow_app.tar.gz lars.holmnielsen@hadoop.openaire.eu:/home/lars.holmnielsen/tmp/

ssh lars.holmnielsen@hadoop.openaire.eu "cd /home/lars.holmnielsen/tmp/; tar -xzvf citation_metrics_advanced_workflow_app.tar.gz; ./upload_workflow.sh; ./run_workflow.sh"

<?xml version="1.0"?>

<!-- Note that documentation placed in comments in this file uses the

"markdown" syntax (along with division into sections). -->

<workflow-app xmlns="uri:oozie:workflow:0.3"

    name="invenio-uploader">

    <start to="invenio_uploader" />

    <action name="invenio_uploader">

        <java>

            <job-tracker>${jobTracker}</job-tracker>

            <name-node>${nameNode}</name-node>

            <!-- The data generated by this node is deleted in this section -->

            <prepare>

                <delete path="${nameNode}${workingDir}/invenio_uploader" />

                <mkdir path="${nameNode}${workingDir}/invenio_uploader" />

            </prepare>

            <configuration>

                <property>

                    <name>mapred.job.queue.name</name>

                    <value>${queueName}</value>

                </property>

            </configuration>

            <!-- This is simple wrapper for the Java code -->

            <main-class>eu.dnetlib.iis.core.java.ProcessWrapper</main-class>

            <!-- The business Java code that gets to be executed -->

            <arg>eu.dnetlib.iis.citationmetrics.InvenioUploader</arg>

            <!-- All input and output ports have to be bound to paths in HDFS, working

                directory has to be specified as well -->

            <arg>-SworkingDir=${workingDir}/invenio_uploader/working_dir</arg>

            <!--<arg>-Operson=${workingDir}/invenio_uploader/person</arg>

            <arg>-Odocument=${workingDir}/invenio_uploader/document</arg>

            -->

        </java>

        <ok to="end" />

        <error to="fail" />

    </action>

    <kill name="fail">

        <message>Unfortunately, the process failed -- error message:

            [${wf:errorMessage(wf:lastErrorNode())}]

        </message>

    </kill>

    <end name="end" />

</workflow-app>

package eu.dnetlib.iis.citationmetrics;

import java.util.HashMap;

import java.util.Map;

import java.io.IOException;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

//import eu.dnetlib.iis.core.examples.schemas.documentandauthor.Person;

//import eu.dnetlib.iis.core.examples.schemas.documentandauthor.Document;

//import eu.dnetlib.iis.core.examples.StandardDataStoreExamples;

import eu.dnetlib.iis.core.java.HadoopContext;

import eu.dnetlib.iis.core.java.PortBindings;

import eu.dnetlib.iis.core.java.Process;

import eu.dnetlib.iis.core.java.io.DataStore;

import eu.dnetlib.iis.core.java.io.FileSystemPath;

import eu.dnetlib.iis.core.java.porttype.AvroPortType;

import eu.dnetlib.iis.core.java.porttype.PortType;

/** Invenio Uploader

 * @author Lars Holm Nielsen

 */

public class InvenioUploader implements Process {

    // private final static String personPort = "person";

    // private final static String documentPort = "document";

    public Map<String, PortType> getInputPorts() {

        return new HashMap<String, PortType>();

    }

    @Override

    public Map<String, PortType> getOutputPorts() {

        return createOutputPorts();

    }

    private static HashMap<String, PortType> createOutputPorts(){

        HashMap<String, PortType> outputPorts =

                new HashMap<String, PortType>();

        // outputPorts.put(personPort,

        //         new AvroPortType(Person.SCHEMA$));

        // outputPorts.put(documentPort,

        //         new AvroPortType(Document.SCHEMA$));

        return outputPorts;

    }

    @Override

    public void run(PortBindings portBindings, HadoopContext context,

            Map<String, String> parameters) throws IOException{

        System.out.println("Hello World...");

        // Map<String, Path> output = portBindings.getOutput();

        // FileSystem fs = FileSystem.get(context.getConfiguration());

        // DataStore.create(StandardDataStoreExamples.getDocument(),

        //         new FileSystemPath(fs, output.get(documentPort)));

        // DataStore.create(StandardDataStoreExamples.getPerson(),

        //         new FileSystemPath(fs, output.get(personPort)));

    }

}

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

    <parent>

        <groupId>eu.dnetlib</groupId>

        <artifactId>icm-iis-parent-container</artifactId>

        <version>0.0.1-SNAPSHOT</version>

    </parent>

    <modelVersion>4.0.0</modelVersion>

    <artifactId>cern-iis-metrics-extended</artifactId>

    <packaging>jar</packaging>

    <!-- <version>0.0.1-SNAPSHOT</version> -->

    <properties>

        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

    </properties>

    <dependencies>

        <dependency>

            <groupId>eu.dnetlib</groupId>

            <artifactId>icm-iis-core</artifactId>

            <version>0.0.1-SNAPSHOT</version>

        </dependency>

        <dependency>

            <groupId>eu.dnetlib</groupId>

            <artifactId>icm-iis-core</artifactId>

            <version>0.0.1-SNAPSHOT</version>

            <type>test-jar</type>

            <scope>test</scope>

        </dependency>

        <dependency>

            <groupId>com.google.protobuf</groupId>

            <artifactId>protobuf-java</artifactId>

            <version>2.4.1</version>

        </dependency>

        <!-- required after introducing 'provided' scope for hadoop libs -->

        <dependency>

			<groupId>org.apache.hadoop</groupId>

			<artifactId>hadoop-common</artifactId>

			<version>${iis.hadoop.common.version}</version>

			<scope>provided</scope>

		</dependency>

        <!-- Needed by Oozie tests { -->

        <dependency>

            <groupId>org.apache.oozie</groupId>

            <artifactId>oozie-core</artifactId>

            <version>${iis.oozie.version}</version>

            <type>test-jar</type>

            <scope>test</scope>

        </dependency>

        <dependency>

            <groupId>org.apache.hadoop</groupId>

            <artifactId>hadoop-hdfs</artifactId>

            <version>${iis.hadoop.hdfs.version}</version>

            <type>test-jar</type>

            <scope>test</scope>

        </dependency>

        <dependency>

            <groupId>org.apache.hadoop</groupId>

            <artifactId>hadoop-test</artifactId>

            <version>${iis.hadoop.test.version}</version>

            <scope>test</scope>

        </dependency>

        <dependency>

            <groupId>org.apache.hadoop</groupId>

            <artifactId>hadoop-common</artifactId>

            <version>${iis.hadoop.common.version}</version>

            <type>test-jar</type>

            <scope>test</scope>

        </dependency>

        <!-- Needed by Oozie tests } -->

        <!-- Needed to run Pig jobs { -->

        <dependency>

            <groupId>org.apache.pig</groupId>

            <artifactId>pig</artifactId>

            <version>${iis.pig.version}</version>

            <scope>test</scope>

        </dependency>

        <dependency>

            <groupId>org.antlr</groupId>

            <artifactId>antlr-runtime</artifactId>

            <version>3.4</version>

        </dependency>

        <!-- Needed to run Pig jobs } -->

        <dependency>

            <groupId>junit</groupId>

            <artifactId>junit</artifactId>

            <version>4.10</version>

            <scope>test</scope>

        </dependency>

        <!-- Needed by Avro { -->

        <dependency>

            <groupId>org.apache.avro</groupId>

            <artifactId>avro</artifactId>

            <version>1.7.4</version>

        </dependency>

        <dependency>

            <groupId>org.apache.avro</groupId>

            <artifactId>avro-mapred</artifactId>

            <version>1.7.4</version>

            <classifier>hadoop2</classifier>

        </dependency>

        <!-- It seems that this dependency is needed, if it is not here,

        a wrong (older) version of this library is used, which is not

        what Avro expects -->

        <dependency>

            <groupId>org.codehaus.jackson</groupId>

            <artifactId>jackson-core-asl</artifactId>

            <version>1.9.11</version>

        </dependency>

        <!-- It seems that this dependency is needed, if it is not here,

        running Oozie tests of Avro-based map-reduce ends up with

        "java.lang.NoClassDefFoundError" exception and statement that

        "Paranamer" class is missing.-->

        <dependency>

            <groupId>com.thoughtworks.paranamer</groupId>

            <artifactId>paranamer</artifactId>

            <version>2.5.2</version>

        </dependency>

        <!-- Needed by Avro } -->

    </dependencies>

    <build>

        <plugins>

            <!-- Plugin that generates Java classes from Avro schemas -->

            <plugin>

                <groupId>org.apache.avro</groupId>

                <artifactId>avro-maven-plugin</artifactId>

                <version>1.7.3</version>

                <executions>

                    <execution>

                        <phase>generate-sources</phase>

                        <goals>

                            <goal>schema</goal>

                            <goal>idl-protocol</goal>

                        </goals>

                        <configuration>

                            <sourceDirectory>${project.basedir}/src/main/resources/</sourceDirectory>

                            <outputDirectory>${project.basedir}/target/generated-sources/java/</outputDirectory>

                        </configuration>

                    </execution>

                </executions>

            </plugin>

            <plugin>

                <groupId>org.apache.maven.plugins</groupId>

                <artifactId>maven-compiler-plugin</artifactId>

                <configuration>

                    <source>1.6</source>

                    <target>1.6</target>

                </configuration>

            </plugin>

            <!-- This plugin makes the Maven->Update Project Configuration not forget

                about the "target/generated-sources/java" source path -->

            <plugin>

                <groupId>org.codehaus.mojo</groupId>

                <artifactId>build-helper-maven-plugin</artifactId>

                <executions>

                    <execution>

                        <id>add-source</id>

                        <phase>generate-sources</phase>

                        <goals>

                            <goal>add-source</goal>

                        </goals>

                        <configuration>

                            <sources>

                                <source>${project.build.directory}/generated-sources/java/</source>

                            </sources>

                        </configuration>

                    </execution>

                </executions>

            </plugin>

        </plugins>

        <pluginManagement>

            <plugins>

                <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->

                <plugin>

                    <groupId>org.eclipse.m2e</groupId>

                    <artifactId>lifecycle-mapping</artifactId>

                    <version>1.0.0</version>

                    <configuration>

                        <lifecycleMappingMetadata>

                            <pluginExecutions>

                                <pluginExecution>

                                    <pluginExecutionFilter>

                                        <groupId>

                                            org.apache.avro

                                        </groupId>

                                        <artifactId>

                                            avro-maven-plugin

                                        </artifactId>

                                        <versionRange>

                                            [1.7.3,)

                                        </versionRange>

                                        <goals>

                                            <goal>idl-protocol</goal>

                                            <goal>schema</goal>

                                        </goals>

                                    </pluginExecutionFilter>

                                    <action>

                                        <ignore></ignore>

                                    </action>

                                </pluginExecution>

                                <pluginExecution>

                                    <pluginExecutionFilter>

                                        <groupId>

                                            org.codehaus.mojo

                                        </groupId>

                                        <artifactId>

                                            build-helper-maven-plugin

                                        </artifactId>

                                        <versionRange>

                                            [1.7,)

                                        </versionRange>

                                        <goals>

                                            <goal>add-source</goal>

                                        </goals>

                                    </pluginExecutionFilter>

                                    <action>

                                        <ignore></ignore>

                                    </action>

                                </pluginExecution>

                            </pluginExecutions>

                        </lifecycleMappingMetadata>

                    </configuration>

                </plugin>

            </plugins>

        </pluginManagement>

    </build>

    <repositories>

        <repository>

            <id>cloudera</id>

            <name>Cloudera Repository</name>

            <url>https://repository.cloudera.com/artifactory/cloudera-repos</url>

            <releases>

                <enabled>true</enabled>

            </releases>

            <snapshots>

                <enabled>false</enabled>

            </snapshots>

        </repository>

        <!-- This repository contains our patched

        version of "avro" and "avro-mapred" modules (see the dependencies section)

        This entry might be removed when the patch to these modules becomes

        a part of the official Avro release.-->

        <repository>

            <id>dnet-deps</id>

            <name>dnet dependencies</name>

            <url>http://maven.research-infrastructures.eu/nexus/content/repositories/dnet-deps</url>

            <releases>

                <enabled>true</enabled>

            </releases>

            <snapshots>

                <enabled>false</enabled>

            </snapshots>

            <layout>default</layout>

        </repository>

    </repositories>

</project>