D-Net

{

  "type_source": "SVN", 

  "goal": "package -U -T 4C source:jar", 

  "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/icm-iis-primer/trunk/", 

  "deploy_repository": "dnet4-snapshots", 

  "version": "4",

  "mail": "m.horst@icm.edu.pl,m.fedoryszak@icm.edu.pl",

  "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-bootstrap-snapshot", 

  "name": "icm-iis-primer"

}

package eu.dnetlib.iis.primer;

import org.junit.*;

import org.junit.rules.TemporaryFolder;

import java.io.File;

import java.io.FileWriter;

import java.io.IOException;

import java.io.Writer;

import java.util.HashSet;

import java.util.Set;

/**

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public class FileClassProviderTest {

    @Rule

    public TemporaryFolder folder = new TemporaryFolder();

    @Test

    public void basicTest() throws IOException, UnresolvableResourceException {

        File sourceDir = folder.newFolder();

        File dirs = new File(sourceDir, "some/random/path");

        if (!dirs.mkdirs()) {

            throw new IOException("Could not create directory " + dirs);

        }

        File f1 = new File(sourceDir, "some/random/f1.txt");

        if (!f1.createNewFile()) {

            throw new IOException("Could not create file " + f1);

        }

        File f2 = new File(sourceDir, "some/random/path/f2.txt");

        if (!f2.createNewFile()) {

            throw new IOException("Could not create file " + f2);

        }

        ClassProvider classProvider = new FileSystemClassProvider(sourceDir);

        Loader loader = new Loader();

        loader.addClassProvider(classProvider);

        File dest = folder.newFolder();

        loader.prime("some/random", dest, new Resolver());

        Assert.assertTrue(new File(dest, "f1.txt").exists());

        Assert.assertTrue(new File(dest, "path/f2.txt").exists());

    }

    @Test

    public void basicImportTest() throws IOException, UnresolvableResourceException {

        File sourceDir = folder.newFolder();

        File dirs1 = new File(sourceDir, "some/random");

        if (!dirs1.mkdirs()) {

            throw new IOException("Could not create directory " + dirs1);

        }

        File importFile = new File(sourceDir, "some/random/import.txt");

        Writer writer = new FileWriter(importFile);

        writer.append(ImportLinkFileChecker.IMPORT_LINK_HEADER + '\n');

        writer.append("path classpath some/other/path/");

        writer.close();

        File dirs2 = new File(sourceDir, "some/other/path");

        if (!dirs2.mkdirs()) {

            throw new IOException("Could not create directory " + dirs2);

        }

        File f1 = new File(sourceDir, "some/random/f1.txt");

        if (!f1.createNewFile()) {

            throw new IOException("Could not create file " + f1);

        }

        File f2 = new File(sourceDir, "some/other/path/f2.txt");

        if (!f2.createNewFile()) {

            throw new IOException("Could not create file " + f2);

        }

        ClassProvider classProvider = new FileSystemClassProvider(sourceDir);

        Loader loader = new Loader();

        loader.addClassProvider(classProvider);

        Resolver resolver = new Resolver();

        resolver.addService("classpath", new ClasspathResolvingService(loader));

        File dest = folder.newFolder();

        loader.prime("some/random", dest, resolver);

        Assert.assertTrue(new File(dest, "f1.txt").exists());

        Assert.assertTrue(new File(dest, "path/f2.txt").exists());

    }

    @Test

    public void fileImportTest() throws IOException, UnresolvableResourceException {

        File sourceDir = folder.newFolder();

        File dirs1 = new File(sourceDir, "some/random/path");

        if (!dirs1.mkdirs()) {

            throw new IOException("Could not create directory " + dirs1);

        }

        File importFile = new File(sourceDir, "some/random/path/import.txt");

        Writer writer = new FileWriter(importFile);

        writer.append(ImportLinkFileChecker.IMPORT_LINK_HEADER + '\n');

        writer.append("f2.txt classpath some/other/path/f2.txt");

        writer.close();

        File dirs2 = new File(sourceDir, "some/other/path");

        if (!dirs2.mkdirs()) {

            throw new IOException("Could not create directory " + dirs2);

        }

        File f1 = new File(sourceDir, "some/random/f1.txt");

        if (!f1.createNewFile()) {

            throw new IOException("Could not create file " + f1);

        }

        File f2 = new File(sourceDir, "some/other/path/f2.txt");

        if (!f2.createNewFile()) {

            throw new IOException("Could not create file " + f2);

        }

        ClassProvider classProvider = new FileSystemClassProvider(sourceDir);

        Loader loader = new Loader();

        loader.addClassProvider(classProvider);

        Resolver resolver = new Resolver();

        resolver.addService("classpath", new ClasspathResolvingService(loader));

        File dest = folder.newFolder();

        loader.prime("some/random", dest, resolver);

        Assert.assertTrue(new File(dest, "f1.txt").exists());

        Assert.assertTrue(new File(dest, "path/f2.txt").exists());

    }

    @Test

    public void getEntryNamesTest() throws IOException {

        File sourceDir = folder.newFolder();

        File dirs1 = new File(sourceDir, "some/random");

        if (!dirs1.mkdirs()) {

            throw new IOException("Could not create directory " + dirs1);

        }

        File importFile = new File(sourceDir, "some/random/import.txt");

        Writer writer = new FileWriter(importFile);

        writer.append(ImportLinkFileChecker.IMPORT_LINK_HEADER + '\n');

        writer.append("path classpath some/other/path/");

        writer.close();

        File dirs2 = new File(sourceDir, "some/other/path");

        if (!dirs2.mkdirs()) {

            throw new IOException("Could not create directory " + dirs2);

        }

        File f1 = new File(sourceDir, "some/random/f1.txt");

        if (!f1.createNewFile()) {

            throw new IOException("Could not create file " + f1);

        }

        File f2 = new File(sourceDir, "some/other/path/f2.txt");

        if (!f2.createNewFile()) {

            throw new IOException("Could not create file " + f2);

        }

        Set entries = new HashSet(new FileSystemClassProvider(sourceDir).getEntryNames());

        Set expected = new HashSet();

        expected.add("some/");

        expected.add("some/random/");

        expected.add("some/random/import.txt");

        expected.add("some/random/f1.txt");

        expected.add("some/other/");

        expected.add("some/other/path/");

        expected.add("some/other/path/f2.txt");

        Assert.assertEquals(expected, entries);

    }

}

package eu.dnetlib.iis.primer;

import org.junit.Assert;

import org.junit.Rule;

import org.junit.Test;

import org.junit.rules.TemporaryFolder;

import java.io.File;

import java.io.IOException;

/**

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public class CoansysWorkflowPackageTest {

    @Rule

    public TemporaryFolder folder = new TemporaryFolder();

    @Test

    public void basicTest() throws IOException {

        File dir = folder.newFolder();

        File packageFile = new File(getClass().getClassLoader()

                .getResource("eu/dnetlib/iis/primer/inner-workflow-1.2-SNAPSHOT-oozie-job.tar.gz").getPath());

        CoansysWorkflowPackage wfPackage = new CoansysWorkflowPackage(packageFile);

        wfPackage.extract(dir);

        Assert.assertEquals(1, dir.list().length);

        Assert.assertTrue(new File(dir, "workflow.xml").exists());

    }

}

package eu.dnetlib.iis.primer;

import org.junit.Assert;

import org.junit.Rule;

import org.junit.Test;

import org.junit.rules.TemporaryFolder;

import java.io.File;

import java.io.FileWriter;

import java.io.IOException;

import java.io.Writer;

/**

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public class MixedClassProviderTest {

    @Rule

    public TemporaryFolder folder = new TemporaryFolder();

    @Test

    public void basicTest() throws IOException, UnresolvableResourceException {

        File sourceDir = folder.newFolder();

        File dirs1 = new File(sourceDir, "some/different");

        if (!dirs1.mkdirs()) {

            throw new IOException("Could not create directory " + dirs1);

        }

        File importFile = new File(sourceDir, "some/different/import.txt");

        Writer writer = new FileWriter(importFile);

        writer.append(ImportLinkFileChecker.IMPORT_LINK_HEADER + '\n');

        writer.append("path classpath some/other/path/");

        writer.close();

        File dirs2 = new File(sourceDir, "some/other/path");

        if (!dirs2.mkdirs()) {

            throw new IOException("Could not create directory " + dirs2);

        }

        File f1 = new File(sourceDir, "some/different/f1.txt");

        if (!f1.createNewFile()) {

            throw new IOException("Could not create file " + f1);

        }

        File f2 = new File(sourceDir, "some/other/path/f2.txt");

        if (!f2.createNewFile()) {

            throw new IOException("Could not create file " + f2);

        }

        ClassProvider fsClassProvider = new FileSystemClassProvider(sourceDir);

        ClassProvider jarClassProvider = new JarClassProvider(new File(

                getClass().getClassLoader().getResource("eu/dnetlib/iis/primer/TestJar.jar").getPath()));

        Loader loader = new Loader();

        loader.addClassProvider(fsClassProvider);

        loader.addClassProvider(jarClassProvider);

        Resolver resolver = new Resolver();

        resolver.addService("classpath", new ClasspathResolvingService(loader));

        File dest = folder.newFolder();

        loader.prime("some/different", dest, resolver);

        Assert.assertTrue(new File(dest, "f1.txt").exists());

        Assert.assertTrue(new File(dest, "path/f2.txt").exists());

        Assert.assertFalse(new File(dest, "path/test/import.txt").exists());

        Assert.assertTrue(new File(dest, "path/test/f3.txt").exists());

        Assert.assertTrue(new File(dest, "path/test/sth/path/f3.txt").exists());

    }

    public void cyclesTest() throws IOException, UnresolvableResourceException {

        File sourceDir = folder.newFolder();

        File dirs1 = new File(sourceDir, "some/random");

        if (!dirs1.mkdirs()) {

            throw new IOException("Could not create directory " + dirs1);

        }

        File importFile = new File(sourceDir, "some/random/import.txt");

        Writer writer = new FileWriter(importFile);

        writer.append(ImportLinkFileChecker.IMPORT_LINK_HEADER + '\n');

        writer.append("path classpath some/other/path/");

        writer.close();

        File dirs2 = new File(sourceDir, "some/other/path");

        if (!dirs2.mkdirs()) {

            throw new IOException("Could not create directory " + dirs2);

        }

        File f1 = new File(sourceDir, "some/random/f1.txt");

        if (!f1.createNewFile()) {

            throw new IOException("Could not create file " + f1);

        }

        File f2 = new File(sourceDir, "some/other/path/f2.txt");

        if (!f2.createNewFile()) {

            throw new IOException("Could not create file " + f2);

        }

        ClassProvider fsClassProvider = new FileSystemClassProvider(sourceDir);

        ClassProvider jarClassProvider = new JarClassProvider(new File(

                getClass().getClassLoader().getResource("eu/dnetlib/iis/primer/TestJar.jar").getPath()));

        Loader loader = new Loader();

        loader.addClassProvider(fsClassProvider);

        loader.addClassProvider(jarClassProvider);

        Resolver resolver = new Resolver();

        resolver.addService("classpath", new ClasspathResolvingService(loader));

        File dest = folder.newFolder();

        loader.prime("some/random", dest, resolver);

        Assert.assertTrue(new File(dest, "f1.txt").exists());

        Assert.assertTrue(new File(dest, "path/f2.txt").exists());

        Assert.assertFalse(new File(dest, "path/test/import.txt").exists());

        Assert.assertTrue(new File(dest, "path/test/f3.txt").exists());

        Assert.assertTrue(new File(dest, "path/test/sth/path/f3.txt").exists());

    }

}

package eu.dnetlib.iis.primer;

import org.apache.commons.io.IOUtils;

import org.junit.Assert;

import org.junit.Rule;

import org.junit.Test;

import org.junit.rules.TemporaryFolder;

import java.io.File;

import java.io.FileOutputStream;

import java.io.IOException;

/**

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public class CoansysResolvingServiceTest {

    @Rule

    public TemporaryFolder folder = new TemporaryFolder();

    @Test

    public void basicTest() throws IOException, UnresolvableResourceException {

        File dir = folder.newFolder();

        File repo = folder.newFolder();

        String filename = "inner-workflow-1.2-SNAPSHOT-oozie-job.tar.gz";

        String path = "eu/dnetlib/iis/primer";

        IOUtils.copy(getClass().getClassLoader().getResourceAsStream(path + "/" + filename),

                new FileOutputStream(new File(repo, filename)));

        Resolver resolver = new Resolver();

        resolver.addService("coansys", new CoansysResolvingService(new FileSystemCoansysWorkflowPackageSupplier(repo)));

        resolver.resolve(dir, "coansys", "pl.edu.icm.coansys:inner-workflow:1.2-SNAPSHOT");

        Assert.assertEquals(1, dir.list().length);

        Assert.assertTrue(new File(dir, "workflow.xml").exists());

    }

    @Test

    public void emptyVersionTest() throws IOException, UnresolvableResourceException {

        File dir = folder.newFolder();

        File repo = folder.newFolder();

        String filename = "inner-workflow-1.2-SNAPSHOT-oozie-job.tar.gz";

        String path = "eu/dnetlib/iis/primer";

        IOUtils.copy(getClass().getClassLoader().getResourceAsStream(path + "/" + filename),

                new FileOutputStream(new File(repo, filename)));

        Resolver resolver = new Resolver();

        resolver.addService("coansys", new CoansysResolvingService(new FileSystemCoansysWorkflowPackageSupplier(repo)));

        resolver.resolve(dir, "coansys", "pl.edu.icm.coansys:inner-workflow");

        Assert.assertEquals(1, dir.list().length);

        Assert.assertTrue(new File(dir, "workflow.xml").exists());

    }

}

package eu.dnetlib.iis.primer;

import org.junit.Assert;

import org.junit.Rule;

import org.junit.Test;

import org.junit.rules.TemporaryFolder;

import java.io.File;

import java.io.IOException;

/**

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public class DuplicatesTest {

    @Rule

    public TemporaryFolder folder = new TemporaryFolder();

    @Test

    public void firstDirTest() throws IOException, UnresolvableResourceException {

        Loader loader = new Loader();

        ClassProvider classProvider1 = new JarClassProvider(new File(

                getClass().getClassLoader().getResource("eu/dnetlib/iis/primer/TestJar.jar").getPath()));

        ClassProvider classProvider2 = new JarClassProvider(new File(

                getClass().getClassLoader().getResource("eu/dnetlib/iis/primer/DuplicateJar.jar").getPath()));

        loader.addClassProvider(classProvider1);

        loader.addClassProvider(classProvider2);

        Resolver resolver = new Resolver();

        resolver.addService("classpath", new ClasspathResolvingService(loader));

        File dest = folder.newFolder();

        loader.prime("", dest, resolver);

        Assert.assertTrue(new File(dest, "some").exists());

        Assert.assertTrue(new File(dest, "some").isDirectory());

    }

    @Test

    public void firstFileTest() throws IOException, UnresolvableResourceException {

        Loader loader = new Loader();

        ClassProvider classProvider1 = new JarClassProvider(new File(

                getClass().getClassLoader().getResource("eu/dnetlib/iis/primer/DuplicateJar.jar").getPath()));

        ClassProvider classProvider2 = new JarClassProvider(new File(

                getClass().getClassLoader().getResource("eu/dnetlib/iis/primer/TestJar.jar").getPath()));

        loader.addClassProvider(classProvider1);

        loader.addClassProvider(classProvider2);

        Resolver resolver = new Resolver();

        resolver.addService("classpath", new ClasspathResolvingService(loader));

        File dest = folder.newFolder();

        loader.prime("", dest, resolver);

        Assert.assertTrue(new File(dest, "some").exists());

        Assert.assertFalse(new File(dest, "some").isDirectory());

    }

}

package eu.dnetlib.iis.primer;

import org.junit.Assert;

import org.junit.Test;

/**

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public class ImportLinkFileCheckerTest {

    @Test

    public void checkHeaderBasicTest() {

        Assert.assertFalse(ImportLinkFileChecker.checkHeaderContents(

                "## ala ma kota"));

        Assert.assertTrue(ImportLinkFileChecker.checkHeaderContents(

                "## This is a classpath-based import file (this header is required)"));

    }

    @Test

    public void checkHeaderUnderlinesTest() {

        Assert.assertTrue(ImportLinkFileChecker.checkHeaderContents(

                "##_This_is_a_classpath_based_import_file_this_header_is_required"));

    }

    @Test

    public void checkHeaderMessInTheBeginningTest() {

        Assert.assertTrue(ImportLinkFileChecker.checkHeaderContents(

                "  ##_This_is_a_classpath_based_import_file_this_header_is_required"));

    }

}

package eu.dnetlib.iis.primer;

import org.junit.Assert;

import org.junit.Rule;

import org.junit.Test;

import org.junit.rules.TemporaryFolder;

import java.io.File;

import java.io.IOException;

/**

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public class JarClassProviderTest {

    @Rule

    public TemporaryFolder folder = new TemporaryFolder();

    @Test

    public void basicTest() throws IOException, UnresolvableResourceException {

        ClassProvider classProvider = new JarClassProvider(new File(

                getClass().getClassLoader().getResource("eu/dnetlib/iis/primer/TestJar.jar").getPath()));

        Loader loader = new Loader();

        loader.addClassProvider(classProvider);

        Resolver resolver = new Resolver();

        resolver.addService("classpath", new ClasspathResolvingService(loader));

        File dest = folder.newFolder();

        loader.prime("some/other/path", dest, resolver);

        Assert.assertFalse(new File(dest, "test/import.txt").exists());

        Assert.assertTrue(new File(dest, "test/f3.txt").exists());

        Assert.assertTrue(new File(dest, "test/sth/path/f3.txt").exists());

    }

}

package eu.dnetlib.iis.primer;

import org.apache.commons.io.filefilter.WildcardFileFilter;

import org.apache.commons.lang.StringUtils;

import java.io.File;

import java.io.FileFilter;

import java.io.FileNotFoundException;

/**

 * Retrieves workflow packages from a local directory.

 *

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public class FileSystemCoansysWorkflowPackageSupplier extends CoansysWorkflowPackageSupplier {

    private File dir;

    private static String ARTIFACT_CLASSIFIER = "oozie-job";

    private static String ARTIFACT_TYPE = "tar.gz";

    public FileSystemCoansysWorkflowPackageSupplier(File dir) {

        this.dir = dir;

    }

    @Override

    public CoansysWorkflowPackage getCoansysWorkflowPackage(String groupId, String artifactId, String version)

            throws FileNotFoundException {

        if(StringUtils.isNotBlank(version)) {

            return new CoansysWorkflowPackage(getWorkflowPackageFile(artifactId, version));

        } else {

            File[] candidates = dir.listFiles((FileFilter) new WildcardFileFilter(getWorkflowPackageFileName(artifactId, "*")));

            if (candidates.length == 0) {

                throw new FileNotFoundException(String.format(

                        "Could not find CoAnSys workflow package matching pattern %s",

                        getWorkflowPackageFile(artifactId, "*").getAbsolutePath()));

            } else if (candidates.length > 1) {

                throw new FileNotFoundException(String.format(

                        "Filename pattern %s is ambiguous. Please specify CoAnSys workflow package version.",

                        getWorkflowPackageFile(artifactId, "*").getAbsolutePath()));

            } else {

                return new CoansysWorkflowPackage(candidates[0]);

            }

        }

    }

    private String getWorkflowPackageFileName(String artifactId, String version) {

        return artifactId + "-" + version + "-" + ARTIFACT_CLASSIFIER + "." + ARTIFACT_TYPE;

    }

    private File getWorkflowPackageFile(String artifactId, String version) {

        return new File(dir, getWorkflowPackageFileName(artifactId, version));

    }

}

package eu.dnetlib.iis.primer;

import java.io.File;

import java.io.IOException;

import java.text.ParseException;

import java.util.List;

/**

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public abstract class ClassProvider {

    /**

     * Returns a list of entries' names available in this class provider. Directory names should have a tailing slash.

     * @return

     */

    public abstract List<String> getEntryNames();

    public abstract void primeOne(String classpath, File destination, Resolver resolver) throws IOException, UnresolvableResourceException, ParseException;

}

package eu.dnetlib.iis.primer;

import java.io.File;

import java.io.IOException;

import java.util.HashMap;

import java.util.Map;

/**

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public class Resolver {

    private Map<String, ResolvingService> services = new HashMap<String, ResolvingService>();

    public void addService(String protocol, ResolvingService r) {

        r.setParent(this);

        services.put(protocol, r);

    }

    public void resolve(File target, String protocol, String args) throws IOException, UnresolvableResourceException {

        ResolvingService r = services.get(protocol);

        if (r == null) {

            throw new RuntimeException("No resolver for protocol " + protocol);

        }

        r.resolve(target, args);

    }

}

package eu.dnetlib.iis.primer;

import org.apache.commons.io.FileUtils;

import org.apache.commons.lang.StringUtils;

import java.io.File;

import java.io.FileInputStream;

import java.io.IOException;

import java.text.ParseException;

import java.util.ArrayList;

import java.util.List;

/**

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public class FileSystemClassProvider extends ClassProvider {

    private File root;

    public FileSystemClassProvider(File root) {

        this.root = root;

    }

    @Override

    public List<String> getEntryNames() {

        List<String> result = new ArrayList<String>();

        appendChildren("", result);

        return result;

    }

    private void appendChildren(String relativePath, List<String> result) {

        for (File f : new File(root, relativePath).listFiles()) {

            String beginning = StringUtils.isNotEmpty(relativePath) ? relativePath : "";

            String newRelativePath = beginning + f.getName();

            if (f.isDirectory()) {

                newRelativePath = newRelativePath + '/';

                appendChildren(newRelativePath, result);

            }

            result.add(newRelativePath);

        }

    }

    @Override

    public void primeOne(String classpath, File destination, Resolver resolver) throws IOException, UnresolvableResourceException, ParseException {

        File src = new File(root, classpath);

        if (src.isDirectory()) {

            FileUtils.forceMkdir(destination);

        } else if (ImportLinkFileChecker.checkName(src)

                && ImportLinkFileChecker.checkHeader(new FileInputStream(src))) {

            new ImportLinkReader(new FileInputStream(src)).performResolution(destination.getParentFile(), resolver);

        } else {

            FileUtils.copyFile(src, destination);

        }

    }

}

package eu.dnetlib.iis.primer;

import org.apache.commons.compress.archivers.tar.TarArchiveEntry;

import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;

import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;

import org.apache.commons.io.FileUtils;

import org.apache.commons.io.IOUtils;

import java.io.*;

/**

 * Workflow package abstraction.

 *

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public class CoansysWorkflowPackage {

    private File file;

    public CoansysWorkflowPackage(File file) throws FileNotFoundException {

        if (!file.exists()) {

            throw new FileNotFoundException("Could not find CoAnSys workflow package " + file.getAbsolutePath());

        }

        this.file = file;

    }

    /**

     * Unpacks a workflow into specified directory. To be more precise: workflow package should contain only a single

     * directory with the workflow. This method will extract its contents into target directory.

     *

     * @param target new workflow root directory

     */

    public void extract(File target) throws IOException {

        BufferedInputStream in = new BufferedInputStream(new FileInputStream(file));

        GzipCompressorInputStream gzIn = new GzipCompressorInputStream(in);

        TarArchiveInputStream tarIn = new TarArchiveInputStream(gzIn);

        TarArchiveEntry entry;

        while ((entry = (TarArchiveEntry)tarIn.getNextEntry()) != null) {

            //The archive should contain only a single directory: we want to unpack its content.

            final File outputFile = new File(target, entry.getName().substring(entry.getName().indexOf('/') + 1));

            if (entry.isDirectory()) {

                FileUtils.forceMkdir(outputFile);

            } else {

                OutputStream outputFileStream = null;

                try {

                    outputFileStream = new FileOutputStream(outputFile);

                    IOUtils.copy(tarIn, outputFileStream);

                } finally {

                    IOUtils.closeQuietly(outputFileStream);

                }

            }

        }

    }

}

package eu.dnetlib.iis.primer;

import org.apache.log4j.Logger;

import java.io.File;

import java.util.*;

/**

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public class Loader {

    private final Logger log = Logger.getLogger(Loader.class);

    private SortedMap<String, ClassProvider> index = new TreeMap<String, ClassProvider>();

    public void addClassProvider(ClassProvider classProvider) {

        List<String> entryNames = classProvider.getEntryNames();

        Collections.sort(entryNames);

        String exclusionPrefix = null;

        for (String s : entryNames) {

            if (exclusionPrefix != null && s.startsWith(exclusionPrefix)) {

                log.warn("Omitting " + s + " as it is a descendant of inconsistently duplicated " + exclusionPrefix);

            } else {

                exclusionPrefix = null;

                if (index.containsKey(toggleTailingSlash(s))) {

                    log.warn("Inconsistent duplicated entries for " + s + ". Will use the first one.");

                    exclusionPrefix = s;

                } else {

                    if (index.containsKey(s)) {

                        log.warn("Duplicated entries for " + s);

                    }

                    index.put(s, classProvider);

                }

            }

        }

    }

    private static String toggleTailingSlash(String s) {

        if (s.endsWith("/")) {

            return s.substring(0, s.length() - 1);

        } else {

            return s + "/";

        }

    }

    public void prime(String classpath, File destination, Resolver resolver)

            throws UnresolvableResourceException {

        boolean didAnything = false;

        SortedMap<String, ClassProvider> tail = index.tailMap(classpath);

        for (Map.Entry<String, ClassProvider> entry : tail.entrySet()) {

            if (!entry.getKey().startsWith(classpath)) {

                break;

            }

            try {

                entry.getValue().primeOne(

                        entry.getKey(),

                        new File(destination, entry.getKey().substring(classpath.length())),

                        resolver);

                didAnything = true;

            } catch (Exception e) {

                throw new UnresolvableResourceException(e);

            }

        }

        if (!didAnything) {

            throw new UnresolvableResourceException("Could not resolve " + classpath + " using the classpath");

        }

    }

}

package eu.dnetlib.iis.primer;

/**

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public class UnresolvableResourceException extends Exception {

    /**

     * Constructs a new exception with <code>null</code> as its detail message.

     * The cause is not initialized, and may subsequently be initialized by a

     * call to {@link #initCause}.

     */

    public UnresolvableResourceException() {

    }

    /**

     * Constructs a new exception with the specified detail message.  The

     * cause is not initialized, and may subsequently be initialized by

     * a call to {@link #initCause}.

     *

     * @param message the detail message. The detail message is saved for

     *                later retrieval by the {@link #getMessage()} method.

     */

    public UnresolvableResourceException(String message) {

        super(message);

    }

    /**

     * Constructs a new exception with the specified detail message and

     * cause.  <p>Note that the detail message associated with

     * <code>cause</code> is <i>not</i> automatically incorporated in

     * this exception's detail message.

     *

     * @param message the detail message (which is saved for later retrieval

     *                by the {@link #getMessage()} method).

     * @param cause   the cause (which is saved for later retrieval by the

     *                {@link #getCause()} method).  (A <tt>null</tt> value is

     *                permitted, and indicates that the cause is nonexistent or

     *                unknown.)

     * @since 1.4

     */

    public UnresolvableResourceException(String message, Throwable cause) {

        super(message, cause);

    }

    /**

     * Constructs a new exception with the specified cause and a detail

     * message of <tt>(cause==null ? null : cause.toString())</tt> (which

     * typically contains the class and detail message of <tt>cause</tt>).

     * This constructor is useful for exceptions that are little more than

     * wrappers for other throwables (for example, {@link

     * java.security.PrivilegedActionException}).

     *

     * @param cause the cause (which is saved for later retrieval by the

     *              {@link #getCause()} method).  (A <tt>null</tt> value is

     *              permitted, and indicates that the cause is nonexistent or

     *              unknown.)

     * @since 1.4

     */

    public UnresolvableResourceException(Throwable cause) {

        super(cause);

    }

}

package eu.dnetlib.iis.primer;

import java.io.File;

import java.io.FileNotFoundException;

import java.io.IOException;

/**

 * This class enables usage of CoAnSys Oozie Workflow Packages.

 *

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public class CoansysResolvingService extends ResolvingService {

    private CoansysWorkflowPackageSupplier packageSupplier;

    public CoansysResolvingService(CoansysWorkflowPackageSupplier packageSupplier) {

        this.packageSupplier = packageSupplier;

    }

    @Override

    public void resolve(File target, String args) throws IOException, UnresolvableResourceException {

        final String[] splits = args.split(":",3);

        final String groupId = splits[0];

        final String artifactId = splits[1];

        final String version = (splits.length == 3) ? splits[2] : null;

        try {

            packageSupplier.getCoansysWorkflowPackage(groupId, artifactId, version).extract(target);

        } catch (FileNotFoundException ex) {

            throw new UnresolvableResourceException("Could not resolve " + args + " CoAnSys workflow package", ex);

        }

    }

}

package eu.dnetlib.iis.primer;

import java.io.File;

import java.io.IOException;

/**

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public abstract class ResolvingService {

    private Resolver parent;

    public Resolver getParent() {

        return parent;

    }

    public void setParent(Resolver parent) {

        this.parent = parent;

    }

    public abstract void resolve(File target, String args) throws IOException, UnresolvableResourceException;

}

package eu.dnetlib.iis.primer;

import java.io.FileNotFoundException;

/**

 * This class retrieves a workflow package given its details.

 *

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public abstract class CoansysWorkflowPackageSupplier {

    public abstract CoansysWorkflowPackage getCoansysWorkflowPackage(String groupId, String artifactId, String version) throws FileNotFoundException;

}

package eu.dnetlib.iis.primer;

import org.apache.commons.lang.StringUtils;

import org.apache.log4j.Logger;

import java.io.*;

import java.text.ParseException;

/**

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public class ImportLinkReader {

    private final Logger log = Logger.getLogger(ImportLinkReader.class);

    private BufferedReader reader;

    public ImportLinkReader(InputStream is) {

        this.reader = new BufferedReader(new InputStreamReader(is));

    }

    public void performResolution(File workingDir, Resolver resolver)

            throws IOException, UnresolvableResourceException, ParseException {

        log.debug("Performing resolution in " + workingDir);

        String line;

        int lineNo = 0;

        ImportLinkFileChecker.dropHeader(reader);

        while ((line = reader.readLine()) != null) {

            ++lineNo;

            if (StringUtils.isBlank(line)) {

                continue;

            }

            final int COLUMN_COUNT = 3;

            String[] splits = line.split("\\s+", COLUMN_COUNT);

            if (splits.length < COLUMN_COUNT) {

                throw new ParseException("Malformed line in " + workingDir + File.separator + "import.txt file", lineNo);

            }

            resolver.resolve(new File(workingDir, splits[0]), splits[1], splits[2]);

        }

    }

}

package eu.dnetlib.iis.primer;

import java.io.File;

import java.io.IOException;

/**

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public class ClasspathResolvingService extends ResolvingService {

    private final Loader loader;

    public ClasspathResolvingService(Loader loader) {

        this.loader = loader;

    }

    @Override

    public void resolve(File target, String args) throws IOException, UnresolvableResourceException {

        loader.prime(args, target, getParent());

    }

}

package eu.dnetlib.iis.primer;

import org.apache.commons.io.IOUtils;

import java.io.*;

import java.util.Arrays;

import java.util.Locale;

/**

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public final class ImportLinkFileChecker {

    public static final String IMPORT_LINK_FILE_NAME = "import.txt";

    public static final String IMPORT_LINK_HEADER =

            "## This is a classpath-based import file (this header is required)";

    private static final String SPLITTING_REGEX = "[^A-Za-z0-9#]+";

    private ImportLinkFileChecker() {}

    /**

     * Checks if a file has a name following import link file rules.

     */

    public static boolean checkName(File f) {

        return f.getName().equalsIgnoreCase(IMPORT_LINK_FILE_NAME);

    }

    /**

     * Removes import link file header from a reader.

     */

    public static void dropHeader(BufferedReader reader) throws IOException {

        reader.readLine();

    }

    /**

     * Checks if a file has a header following import link file rules.

     */

    public static boolean checkHeader(InputStream is) {

        BufferedReader reader = null;

        try {

            reader = new BufferedReader(new InputStreamReader(is));

            String header = reader.readLine();

            return checkHeaderContents(header);

        } catch (IOException e) {

            return false;

        } finally {

            IOUtils.closeQuietly(reader);

        }

    }

    /**

     * Checks if header contents follow import link file rules.

     */

    public static boolean checkHeaderContents(String header) {

        String[] actual = splitHeader(header);

        String[] expected = splitHeader(IMPORT_LINK_HEADER);

        return Arrays.equals(actual, expected);

    }

    private static String[] splitHeader(String header) {

        return header.trim().toLowerCase(Locale.ENGLISH).split(SPLITTING_REGEX);

    }

}

package eu.dnetlib.iis.primer;

import org.apache.commons.io.FileUtils;

import org.apache.commons.io.IOUtils;

import java.io.File;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.InputStream;

import java.text.ParseException;

import java.util.ArrayList;

import java.util.Enumeration;

import java.util.List;

import java.util.jar.JarEntry;

import java.util.jar.JarFile;

/**

 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)

 */

public class JarClassProvider extends ClassProvider {

    private JarFile jar;

    public JarClassProvider(File jar) throws IOException {

        this.jar = new JarFile(jar);

    }

    @Override

    public List<String> getEntryNames() {

        List<String> result = new ArrayList<String>();

        Enumeration<JarEntry> entries = jar.entries();

        while (entries.hasMoreElements()) {

            final JarEntry entry = entries.nextElement();

            result.add(entry.getName());

        }

        return result;

    }

    @Override

    public void primeOne(String classpath, File target, Resolver resolver) throws IOException, UnresolvableResourceException, ParseException {

        final JarEntry entry = jar.getJarEntry(classpath);

        if (entry.isDirectory()) {

            FileUtils.forceMkdir(target);

        } else if (ImportLinkFileChecker.checkName(new File(entry.getName()))

                && ImportLinkFileChecker.checkHeader(jar.getInputStream(entry))) {

            new ImportLinkReader(jar.getInputStream(entry)).performResolution(target.getParentFile(), resolver);

        } else {

            FileUtils.forceMkdir(target.getParentFile());

            FileOutputStream output = null;

            try {

                output = new FileOutputStream(target);

                InputStream input = jar.getInputStream(entry);

                IOUtils.copy(input, output);

            } finally {

                IOUtils.closeQuietly(output);

            }

        }

    }

}

log4j.rootLogger=DEBUG, A1

log4j.appender.A1=org.apache.log4j.ConsoleAppender

log4j.appender.A1.layout=org.apache.log4j.PatternLayout

# Print the date in ISO 8601 format

log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c - %m%n

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

    <modelVersion>4.0.0</modelVersion>

    <groupId>eu.dnetlib</groupId>

    <artifactId>icm-iis-primer</artifactId>

    <version>0.0.1</version>

    <packaging>jar</packaging>

    <name>icm-iis-classpath-primer</name>

    <url>http://maven.apache.org</url>

    <scm>

       <developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/icm-iis-primer/tags/icm-iis-primer-0.0.1</developerConnection>

    </scm>

    <dependencies>

        <dependency>

            <groupId>junit</groupId>

            <artifactId>junit</artifactId>

            <version>4.10</version>

        </dependency>

        <dependency>

            <groupId>commons-io</groupId>

            <artifactId>commons-io</artifactId>

            <version>2.4</version>

        </dependency>

        <dependency>

            <groupId>log4j</groupId>

            <artifactId>log4j</artifactId>

            <version>1.2.17</version>

        </dependency>

        <dependency>

            <groupId>org.apache.commons</groupId>

            <artifactId>commons-compress</artifactId>

            <version>1.4.1</version>

        </dependency>

        <dependency>

            <groupId>commons-lang</groupId>

            <artifactId>commons-lang</artifactId>

            <version>2.6</version>

        </dependency>

    </dependencies>

    <properties>

        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

    </properties>

    <build>

        <plugins>

            <plugin>

                <groupId>org.apache.maven.plugins</groupId>

                <artifactId>maven-compiler-plugin</artifactId>

                <version>3.1</version>

                <configuration>

                    <source>1.5</source>

                    <target>1.5</target>

                </configuration>

            </plugin>

        </plugins>

    </build>

    <distributionManagement>

	<repository>

	<id>dnet4-bootstrap-release</id>

	<url>

	http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-bootstrap-release

	</url>

	</repository>

    </distributionManagement>

</project>

icm-iis-primer

==============

The aim of this project is to provide a reusability method for Oozie workflows, 

scripts and other file types by utilising import links.

The Primer traverses a specified Java package, looking for import links. When 

found, it resolves them by creating new elements in the containing directory,

in accordance with the import link. The primed package is stored in a new 

directory.

## Import links ##

Import links are defined as files following the specified format. They are 

always named `import.txt`.

### Exemplary import link

    ## This is a classpath-based import file (this header is required)

    from_jar classpath pl/edu/icm/coansys

    workflow coansys pl.edu.icm.coansys.citations:inner-workflow:1.2-SNAPSHOT

The file starts with a compulsory header. Then, every line consists of the 

following fields separated by whitespaces:

 - destination - the name that will be given to the imported element

 - import type - the type of imported element

 - imported element description - additional information indentifying the 

   imported element (import type-specific)

The import file presented above will create a directory `from_jar` and 

copy all the contents of `pl/edu/icm/coansys` into it. Additionally, it will 

unpack `inner-workflow` into `workflow` directory. 

### Supported import types ###

#### Classpath element ####

The element will be imported from the classpath (it can be either a file or 

directory).

 - **Import type:** `classpath`

 - **Element description:** fully qualified resource name, using forward slashes 

   as separators.

#### CoAnSys Oozie Workflow Package ####

A CoAnSys Oozie Workflow Package will be imported and unpacked.

 - **Import type:** `coansys`

 - **Element description:** Maven-style package description, i.e. 

   `groupId:artifactId:version`

## Execution ##

Primer is executed from a Maven plugin implemented in 

`icm-iis-primer-maven-plugin` project. Plugin usage is presented in 

`icm-iis-primer-example` project.

## For developer ##

The main element of Primer is `Loader` class. It is a close relative of 

Java `ClassLoader`. It should be provided with `ClassProviders`, which tell 

where it should look for resources. By default, `JarClassProvider` and 

`FileSystemClassProvider` are supplied. `Loader.prime` method executes the 

priming process.

Another important class is `Resolver`. It is used when resolving import links.

It groups various `ResolvingServices`. Each import type should be assigned a

`ResolvingService`. Currently implemented `ResolvingServices` are 

`ClasspathResolvingService` and `CoansysResolvingService`.