D-Net

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-download-plugins/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-download-plugins"}

import org.joda.time.DateTime;

import org.joda.time.Days;

import org.junit.Assert;

import org.junit.Before;

import org.junit.Test;

public class DateIntervalTest {

	@Before

	public void setUp() throws Exception {}

	@Test

	public void test() {

		DateTime beforeDate = new DateTime(2010, 01, 1, 0, 0);

		DateTime now = new DateTime();

		Days day = Days.daysBetween(beforeDate, now);

		Assert.assertTrue(day.getDays() > 0);

	}

}

package eu.dnetlib.download.plugin;

import junit.framework.Assert;

import org.junit.Before;

import org.junit.Test;

import com.google.gson.Gson;

import com.google.gson.GsonBuilder;

import eu.dnetlib.data.download.rmi.DownloadItem;

public class ArxivImportFromFileTest {

	@Before

	public void setUp() throws Exception {}

	@Test

	public void testREgEx() {

		String regExp = "\\d{4}\\.\\d{4}";

		String input = "1308.0021";

		Assert.assertTrue(input.matches(regExp));

	}

	@Test

	public void testPlugin() {

		DownloadItem item = new DownloadItem();

		item.setOpenAccess("OPEN");

		String[] myList = new String[] { "http://arxiv.org/abs/1308.0001", "b" };

		Gson g = new GsonBuilder().disableHtmlEscaping().create();

		item.setUrl(g.toJson(myList));

		ArxivImportFromFile plugin = new ArxivImportFromFile();

		plugin.setBasePath("/test/path");

		plugin.retrieveUrl(item);

		Assert.assertNotNull(item);

	}

}

package eu.dnetlib.download.plugin;

import org.junit.Assert;

import org.junit.Ignore;

import org.junit.Test;

/**

 * Created by sandro on 5/15/17.

 */

@Ignore

public class DSpacePDFLinkPluginsTest {

    @Test

    public void testURL () {

        final DSpacePDFLinkPlugins plugin = new DSpacePDFLinkPlugins();

        final String s1 = plugin.extractURL("http://acikerisim.baskent.edu.tr/handle/11727/2340");

        Assert.assertEquals(s1, "http://acikerisim.baskent.edu.tr/bitstream/11727/2340/1/10008302.pdf");

    }

    @Test

    public void testDOI() {

        final DSpacePDFLinkPlugins plugin = new DSpacePDFLinkPlugins();

        final String s1 = plugin.extractURL("http://hdl.handle.net/10017/29866");

       // Assert.assertEquals(s1, "http://acikerisim.baskent.edu.tr/bitstream/11727/2340/1/10008302.pdf");

        System.out.println(s1);

    }

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

// TODO: Auto-generated Javadoc

/**

 * The Class EuropePMC.

 */

public class EuropePMC extends AbstractDownloadPlugin implements DownloadPlugin {

    private static final Log log = LogFactory.getLog(EuropePMC.class);

    /**

     * The base path.

     */

    private String basePath;

    // //*[local-name()='metadata']//*[local-name()='identifier' and ./@identifierType='pmc']/text()

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "europePMCDownloadPlugin";

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) throws DownloadPluginException {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) throws DownloadPluginException {

        try {

            String url = input.getOriginalUrl();

            if ((url == null) || (url.trim().length() == 0)) return input;

            @SuppressWarnings("unchecked")

            List<String> urls = new Gson().fromJson(url, ArrayList.class);

            log.debug(String.format("urls is %s", url));

            if ((urls == null) || (urls.size() == 0)) return input;

            input.setFileName(input.getIdItemMetadata());

            Boolean added = false;

            for (String s : urls) {

                if (s.startsWith("http")) {

                    log.debug(String.format("found url starting with http replace original URL with %s", s));

                    input.setOriginalUrl(s);

                } else if (s.startsWith("PMC")) {

                    String correctUrl = s.replace("PMC", "");

                    log.debug(String.format("found url starting with PMC %s", correctUrl));

                    added = true;

                    String path = PathRetreiver.getInstance(basePath).getPathForPMCID(Integer.parseInt(correctUrl));

                    if (path != null) {

                        input.setUrl("file://" + path);

                    } else {

                        input.setUrl(path);

                    }

                }

            }

            if (added == false) {

                input.setOriginalUrl(null);

                input.setUrl(null);

            }

            return input;

        } catch (Throwable e) {

            log.error("Exception on Download Plugin");

            log.error(e);

            throw new DownloadPluginException(e);

        }

    }

    /**

     * Gets the base path.

     *

     * @return the basePath

     */

    public String getBasePath() {

        return basePath;

    }

    /**

     * Sets the base path.

     *

     * @param basePath the basePath to set

     */

    @Override

    public void setBasePath(final String basePath) {

        this.basePath = basePath;

    }

    @Override

    public String extractURL(String baseURL) throws DownloadPluginException {

        return null;

    }

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

public class ArxivDownloadPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        if (checkOpenAccess(input) == null) {

            return null;

        }

        String url = input.getOriginalUrl();

        if (url == null || url.trim().length() == 0) {

            return input;

        }

        @SuppressWarnings("unchecked")

        List<String> urls = new Gson().fromJson(url, ArrayList.class);

        if (urls == null || urls.size() == 0) {

            return input;

        }

        for (String s : urls) {

            if (s.startsWith("http")) {

                input.setOriginalUrl(s);

                String correctUrl = s.replace("abs", "pdf");

                correctUrl += ".pdf";

                input.setUrl(correctUrl);

            }

        }

        return input;

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    @Override

    public String getPluginName() {

        return "ArxivDownloadPlugin";

    }

    @Override

    public void setBasePath(final String basePath) {

    }

    @Override

    public String extractURL(String baseURL) throws DownloadPluginException {

        return null;

    }

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

/**

 * The Class EasyPDFDownloadPlugin.

 */

public class EasyPDFDownloadPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        if (checkOpenAccess(input) == null) {

            return null;

        }

        if (input == null) {

            return null;

        }

        String url = input.getOriginalUrl();

        if (url == null || url.trim().length() == 0) {

            return input;

        }

        @SuppressWarnings("unchecked")

        List<String> urls = new Gson().fromJson(url, ArrayList.class);

        if (urls == null || urls.size() == 0) {

            return input;

        }

        if (checkUrlsNotNull(input, urls))

            return input;

        input.setOriginalUrl(null);

        input.setUrl(null);

        return input;

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "easyPDFDownloadPlugin";

    }

    @Override

    public void setBasePath(final String basePath) {

        // TODO Auto-generated method stub

    }

    @Override

    public String extractURL(String baseURL) throws DownloadPluginException {

        return baseURL.endsWith(".pdf") ? baseURL : null;

    }

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

/**

 * The Class ELisDownloadPlugin.

 */

public class ELisDownloadPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

    /**

     * The Constant log.

     */

    private static final Log log = LogFactory.getLog(ELisDownloadPlugin.class);

    /**

     * Extract url.

     *

     * @param url the url

     * @return the string

     */

    @Override

    public String extractURL(final String url) throws DownloadPluginException {

        try {

            Document doc = Jsoup.connect(url).get();

            Elements links = doc.select("a[href$=.pdf]");

            for (Element link : links) {

                String linkvalue = link.attr("abs:href");

                if (!linkvalue.toLowerCase().contains("thumbnailversion")) {

                    return linkvalue;

                }

            }

            return null;

        } catch (Exception e) {

            throw new DownloadPluginException("Error on extract URL", e);

        }

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "ELisDownloadPlugin";

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        if (checkOpenAccess(input) == null) {

            return null;

        }

        String url = input.getOriginalUrl();

        if (url == null || url.trim().length() == 0) {

            return input;

        }

        @SuppressWarnings("unchecked")

        List<String> urls = new Gson().fromJson(url, ArrayList.class);

        if (urls == null || urls.size() == 0) {

            return input;

        }

        if (checkUrlsNotNull(input, urls))

            return input;

        input.setOriginalUrl(null);

        input.setUrl(null);

        return input;

    }

    @Override

    public void setBasePath(final String basePath) {

        // TODO Auto-generated method stub

    }

}

package eu.dnetlib.download.plugin;

import java.io.File;

import java.io.FileFilter;

import java.util.ArrayList;

import java.util.Collections;

import java.util.Comparator;

import java.util.List;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

// TODO: Auto-generated Javadoc

/**

 * The Class PathRetreiver.

 */

public class PathRetreiver {

	private static final Log log = LogFactory.getLog(PathRetreiver.class);

	/** The instance. */

	private static PathRetreiver instance;

	/**

	 * Gets the single instance of PathRetreiver.

	 *

	 * @param base_path

	 *            the base_path

	 * @return single instance of PathRetreiver

	 */

	public static PathRetreiver getInstance(final String base_path) {

		if (instance == null) {

			instance = new PathRetreiver();

			instance.setBase_path(base_path);

		}

		return instance;

	}

	/** The base_path. */

	private String base_path;

	/** The values. */

	private List<InfoPath> values;

	/**

	 * Bootstrap.

	 */

	private void bootstrap() {

		values = new ArrayList<InfoPath>();

		File basePath = new File(this.base_path);

		File[] selectedFiles = basePath.listFiles(new FileFilter() {

			@Override

			public boolean accept(final File pathname) {

				return pathname.isDirectory();

			}

		});

		for (File f : selectedFiles) {

			String lower = StringUtils.substringAfter(StringUtils.substringBefore(f.getName(), "_"), "PMC");

			String upper = StringUtils.substringAfter(StringUtils.substringAfter(f.getName(), "_"), "PMC");

			String path = f.getPath();

			InfoPath i = new InfoPath();

			i.setLower(Integer.parseInt(lower));

			i.setUpper(Integer.parseInt(upper));

			i.setPath(path);

			values.add(i);

		}

		Collections.sort(values, new Comparator<InfoPath>() {

			@Override

			public int compare(final InfoPath o1, final InfoPath o2) {

				if (o1.getLower() < o2.getLower()) return -1;

				else if (o1.getLower() < o2.getLower()) return 0;

				else return 1;

			}

		});

		if (log.isDebugEnabled()) {

			for (InfoPath p : values) {

				log.debug(String.format("%s -- %s : %s", p.getLower(), p.getUpper(), p.getPath()));

			}

		}

	}

	/**

	 * Gets the path for pmcid.

	 *

	 * @param pmcID

	 *            the pmc id

	 * @return the path for pmcid

	 */

	public String getPathForPMCID(final int pmcID) {

		if (values == null) {

			bootstrap();

		}

		for (int i = 0; i < values.size(); i++) {

			if (pmcID < values.get(i).getLower()) {

				if (i == 0) return null;

				String currentPath = values.get(i - 1).getPath() + "/" + pmcID + ".xml";

				File f = new File(currentPath);

				log.debug(String.format("try to search in path %s", currentPath));

				String s = null;

				if (f.exists()) {

					s = f.getPath();

					log.debug(String.format("found in %s", s));

				} else {

					log.debug(String.format("not found in %s", s));

				}

				return s;

			}

		}

		log.debug(String.format("PMC with ID: %s not found", pmcID));

		return null;

	}

	/**

	 * Sets the base_path.

	 *

	 * @param base_path

	 *            the new base_path

	 */

	public void setBase_path(final String base_path) {

		this.base_path = base_path;

	}

	/**

	 * Gets the base_path.

	 *

	 * @return the base_path

	 */

	public String getBase_path() {

		return this.base_path;

	}

}

package eu.dnetlib.download.plugin;

import java.io.File;

import java.nio.file.Path;

import java.nio.file.Paths;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

public class ArxivImportFromFile extends AbstractDownloadPlugin implements DownloadPlugin {

    private static final Log log = LogFactory.getLog(ArxivImportFromFile.class); // NOPMD by marko on 11/24/08 5:02 PM

    /**

     * The base path.

     */

    private String basePath;

    @Override

    public DownloadItem retrieveUrl(final DownloadItem item) {

        if (checkOpenAccess(item) == null) return null;

        final String baseURLs = item.getUrl();

        final List<String> urlsList = new Gson().fromJson(baseURLs, List.class);

        for (final String baseURL : urlsList) {

            if (baseURL.isEmpty() == false && baseURL.trim().startsWith("http://") == true) {

                final String name = StringUtils.substringAfter(baseURL, "abs/").trim();

                if (name == null) {

                    item.setUrl(null);

                    return item;

                }

                final String fileURL = createPath(name);

                if (StringUtils.isBlank(fileURL)) {

                    item.setUrl(null);

                    return item;

                }

                final File f = new File(fileURL);

                if (f.exists()) {

                    if (log.isDebugEnabled()) {

                        log.debug("found path associated to " + item.getIdItemMetadata() + " with path : " + fileURL);

                    }

                    item.setUrl("file://" + fileURL);

                } else {

                    if (log.isDebugEnabled()) {

                        log.debug("NOT found path associated to " + item.getIdItemMetadata());

                    }

                    item.setUrl(null);

                }

                item.setOriginalUrl(baseURL);

                return item;

            } else {

                item.setUrl(null);

            }

        }

        return item;

    }

    public String createPath(final String name) {

        final String regExp = "\\d{4}\\.\\d{4}";

        if (name.matches(regExp)) {

            final String[] values = name.split("\\.");

            final Path bsPath = Paths.get(basePath);

            final Path filePath = Paths.get(String.format("%s/%s.pdf", values[0], name));

            final String fileURL = bsPath.resolve(filePath).toString();

            return fileURL;

        } else {

            if (name.contains("/")) {

                final String[] values = name.split("/");

                if (values.length != 2) return null;

                if (values[1].length() > 4) {

                    final String middle = values[1].substring(0, 4);

                    final Path bsPath = Paths.get(basePath);

                    final Path filePath = Paths.get(String.format("%s/%s.pdf", middle, name.replace("/", "")));

                    final String fileURL = bsPath.resolve(filePath).toString();

                    return fileURL;

                }

            }

        }

        return null;

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> items) {

        return Iterables.transform(items, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    @Override

    public String getPluginName() {

        return "ArxivImportFromFile";

    }

    /**

     * @return the basePath

     */

    public String getBasePath() {

        return basePath;

    }

    /**

     * @param basePath the basePath to set

     */

    @Override

    public void setBasePath(final String basePath) {

        this.basePath = basePath;

    }

    @Override

    public String extractURL(String baseURL) throws DownloadPluginException {

        return null;

    }

}

package eu.dnetlib.download.plugin;

/**

 * The Class InfoPath.

 */

public class InfoPath {

	/** The lower. */

	private int lower;

	/** The upper. */

	private int upper;

	/** The path. */

	private String path;

	/**

	 * Gets the lower.

	 *

	 * @return the lower

	 */

	public int getLower() {

		return lower;

	}

	/**

	 * Sets the lower.

	 *

	 * @param lower

	 *            the new lower

	 */

	public void setLower(final int lower) {

		this.lower = lower;

	}

	/**

	 * Gets the upper.

	 *

	 * @return the upper

	 */

	public int getUpper() {

		return upper;

	}

	/**

	 * Sets the upper.

	 *

	 * @param upper

	 *            the new upper

	 */

	public void setUpper(final int upper) {

		this.upper = upper;

	}

	/**

	 * Gets the path.

	 *

	 * @return the path

	 */

	public String getPath() {

		return path;

	}

	/**

	 * Sets the path.

	 *

	 * @param path

	 *            the new path

	 */

	public void setPath(final String path) {

		this.path = path;

	}

}

/**

 *

 */

package eu.dnetlib.download.plugin;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

/**

 * The Class EasyPDFDownloadPlugin.

 */

public class DLibPlugin extends HALPdfDocumentPlugin {

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "DLIBDownloadPlugin";

    }

    @Override

    public void setBasePath(final String basePath) {

        // TODO Auto-generated method stub

    }

    @Override

    public String extractURL(String baseURL) throws DownloadPluginException {

        return baseURL.trim().endsWith(".html") ? baseURL : null;

    }

}

package eu.dnetlib.download.plugin;

import java.io.IOException;

import java.net.HttpURLConnection;

import java.net.URL;

import java.util.ArrayList;

import java.util.List;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

public class DSpacePDFLinkPlugins extends AbstractDownloadPlugin implements DownloadPlugin {

    /**

     * The Constant log.

     */

    private static final Log log = LogFactory.getLog(DSpacePDFLinkPlugins.class);

    private final static int maxNumberJump = 10;

    private final static int maxNumberConnectRetries = 5;

    /**

     * Milliseconds used to backoff in case of connection errors.

     */

    private final static int BACKOFF_FACTOR = 100;

    private String getHTTPRedirectedURL(final String mainURL) throws Exception {

        URL startURL = new URL(mainURL);

        HttpURLConnection conn = (HttpURLConnection) startURL.openConnection();

        conn.setConnectTimeout(AbstractDownloadPlugin.DEFAULT_TIMEOUT);

        conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manually.

        HttpURLConnection.setFollowRedirects(true);

        String location = mainURL;

        int numJump = 1;

        int responseCode = conn.getResponseCode();

        while ((responseCode >= 300) && (responseCode < 400) && (numJump++ < maxNumberJump)) {

            location = conn.getHeaderFields().get("Location").get(0);

            conn.disconnect();

            startURL = new URL(location);

            conn = (HttpURLConnection) startURL.openConnection();

            conn.setConnectTimeout(AbstractDownloadPlugin.DEFAULT_TIMEOUT);

            conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manually.

            HttpURLConnection.setFollowRedirects(true);

            responseCode = conn.getResponseCode();

        }

        conn.disconnect();

        if (!((responseCode >= 200) && (responseCode < 300)))

            return null;

        return location;

    }

    /**

     * Extract url.

     *

     * @param url the url

     * @return the string

     */

    @Override

    public String extractURL(final String url) throws DownloadPluginException {

        try {

            final String location = getHTTPRedirectedURL(url);

            if (location == null) {

                return null;

            }

            Document doc = null;

            int retries = 0;

            boolean success = false;

            while(retries < maxNumberConnectRetries) {

                try {

                    doc = Jsoup.connect(location).timeout(AbstractDownloadPlugin.DEFAULT_TIMEOUT).get();

                    success = true;

                    break;

                } catch (IOException e) {

                    final int millis = BACKOFF_FACTOR * (retries + 1);

                    log.debug(String.format("backoff for %s ms before retrying on %s", millis, location));

                    Thread.sleep(millis);

                }

                retries++;

            }

            if (!success) {

                throw new DownloadPluginException("reached max number of connect retries for URL: " + location);

            }

            final Elements links = doc.select("meta[content]");

            for (Element link : links) {

                String linkValue = link.attr("content");

                if (regularExpression != null) {

                    for (String regex : regularExpression) {

                        if (linkValue.matches(regex)) {

                            return linkValue;

                        }

                    }

                } else {

                    return linkValue;

                }

            }

            return null;

        } catch (Throwable e) {

	        throw new DownloadPluginException("Error on extract URL", e);

        }

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, input -> retrieveUrl(input));

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "DSpacePDFLinkPlugins";

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        if (checkOpenAccess(input) == null) return null;

        String url = input.getOriginalUrl();

        if ((url == null) || (url.trim().length() == 0)) return input;

        @SuppressWarnings("unchecked")

        List<String> urls = new Gson().fromJson(url, ArrayList.class);

        if ((urls == null) || (urls.size() == 0)) return input;

        if (checkUrlsNotNull(input, urls))

            return input;

        input.setOriginalUrl(null);

        input.setUrl(null);

        return input;

    }

    @Override

    public void setBasePath(final String basePath) {

        // TODO Auto-generated method stub

    }

}

package eu.dnetlib.download.plugin;

import java.nio.file.FileSystems;

import java.nio.file.Files;

import java.nio.file.Path;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

/**

 * Created by sandro on 3/1/16.

 */

public class HindawiDownloadPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

	private String basePath;

	private Gson gson;

	public HindawiDownloadPlugin() {

		this.gson = new Gson();

	}

	@Override

	public String extractURL(final String baseURL) throws DownloadPluginException {

		return null;

	}

	@Override

	public void setBasePath(final String basePath) {

		this.basePath = basePath;

	}

	/*

	 * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

	@Override

	public String getPluginName() {

		return "HindawiDownloadPlugin";

	}

	@Override

	public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) throws DownloadPluginException {

		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

			@Override

			public DownloadItem apply(final DownloadItem input) {

				return retrieveUrl(input);

			}

		});

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

	 */

	@Override

	public DownloadItem retrieveUrl(final DownloadItem input) throws DownloadPluginException {

		List<String> inputList = this.gson.fromJson(input.getUrl(), ArrayList.class);

		if (inputList == null || inputList.size() == 0) {

			input.setUrl(null);

			return input;

		}

		for (String fileUrl : inputList) {

			Path inputhPath = FileSystems.getDefault().getPath(basePath).resolve("." + fileUrl);

			if (Files.exists(inputhPath)) {

				input.setOriginalUrl("file://" + inputhPath.toString());

				input.setUrl("file://" + inputhPath.toString());

				return input;

			}

		}

		input.setUrl(null);

		return input;

	}

}

package eu.dnetlib.download.plugin;

import java.net.HttpURLConnection;

import java.net.URL;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

public class FollowPDFLinkPlugins extends AbstractDownloadPlugin implements DownloadPlugin {

    /**

     * The Constant log.

     */

    private static final Log log = LogFactory.getLog(FollowPDFLinkPlugins.class);

    private final static int maxNumberJump = 10;

    /**

     * Extract url.

     *

     * @param url the url

     * @return the string

     */

    @Override

    public String extractURL(final String url) throws DownloadPluginException {

        try {

            URL startURL = new URL(url);

            HttpURLConnection conn = (HttpURLConnection) startURL.openConnection();

	        conn.setConnectTimeout(AbstractDownloadPlugin.DEFAULT_TIMEOUT);

            conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manully.

            HttpURLConnection.setFollowRedirects(true);

            String location = url;

            int numJump =1;

            int responseCode = conn.getResponseCode();

            while ((responseCode >= 300) && (responseCode < 400) &&  (numJump++ < maxNumberJump)) {

                location = conn.getHeaderFields().get("Location").get(0);

                conn.disconnect();

                startURL = new URL(location);

                conn = (HttpURLConnection) startURL.openConnection();

                conn.setConnectTimeout(AbstractDownloadPlugin.DEFAULT_TIMEOUT);

                conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manully.

                HttpURLConnection.setFollowRedirects(true);

                responseCode = conn.getResponseCode();

            }

            conn.disconnect();

            if (!((responseCode >= 200) && (responseCode < 300)))

                return null;

            Document doc = Jsoup.connect(location).get();

            Elements links = doc.select("a[href$=.pdf]");

            for (Element link : links) {

                String linkValue = link.attr("abs:href");

                if (regularExpression != null) {

                    for (String regex : regularExpression) {

                        if (linkValue.matches(regex))

                            return linkValue;

                    }

                } else

                    return linkValue;

            }

            return null;

        } catch (Throwable e) {

	        throw new DownloadPluginException("Error on extract URL", e);

        }

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "FollowPDFLinkPlugins";

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        if (checkOpenAccess(input) == null) return null;

        String url = input.getOriginalUrl();

        if ((url == null) || (url.trim().length() == 0)) return input;

        @SuppressWarnings("unchecked")

        List<String> urls = new Gson().fromJson(url, ArrayList.class);

        if ((urls == null) || (urls.size() == 0)) return input;

        if (checkUrlsNotNull(input, urls))

            return input;

        input.setOriginalUrl(null);

        input.setUrl(null);

        return input;

    }

    @Override

    public void setBasePath(final String basePath) {

        // TODO Auto-generated method stub

    }

}

/**

 *

 */

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * @author sandro

 */

public class HALPdfDocumentPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

    private static final Log log = LogFactory.getLog(HALPdfDocumentPlugin.class); // NOPMD by marko on 11/24/08 5:02 PM

    /**

     * {@inheritDoc}

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        // TODO Auto-generated method stub

        return "HALPdfDocumentPlugin";

    }

    /**

     * {@inheritDoc}

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrls(java.lang.Iterable)

     */

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /**

     * {@inheritDoc}

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        try {

            if (checkOpenAccess(input) == null) return null;

            if (input == null) return null;

            String url = input.getOriginalUrl();

            if ((url == null) || (url.trim().length() == 0)) return input;

            @SuppressWarnings("unchecked")

            List<String> urls = new Gson().fromJson(url, ArrayList.class);

            if ((urls == null) || (urls.size() == 0)) return input;

            if (checkUrlsNotNull(input, urls)) return input;

            input.setOriginalUrl(null);

            input.setUrl(null);

            return input;

        } catch (Throwable e) {

            log.error("Error on retreiving URL", e);

            input.setOriginalUrl(null);

            input.setUrl(null);

            return input;

        }

    }

    /**

     * {@inheritDoc}

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#setBasePath(java.lang.String)

     */

    @Override

    public void setBasePath(final String arg0) {

        // TODO Auto-generated method stub

    }

    @Override

    public String extractURL(String baseURL) throws DownloadPluginException {

        return baseURL.toLowerCase().endsWith("document") ? baseURL : null;

    }

}

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

       xmlns:http="http://cxf.apache.org/transports/http/configuration"

       xmlns="http://www.springframework.org/schema/beans"

       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd

                                    http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd">

	<bean id= "easyPDFDownloadPlugin" class="eu.dnetlib.download.plugin.EasyPDFDownloadPlugin"/>

	<bean id= "arxivDownloadPlugin" class="eu.dnetlib.download.plugin.ArxivDownloadPlugin"/>

	<bean id= "europePMCDownloadPlugin" class="eu.dnetlib.download.plugin.EuropePMC"/>

	<bean id= "elisDownloadPlugin" class="eu.dnetlib.download.plugin.ELisDownloadPlugin"/>

	<bean id="followPDFLinkPlugins" class="eu.dnetlib.download.plugin.FollowPDFLinkPlugins"/>

	<bean id="ArxivImportFromFile" class="eu.dnetlib.download.plugin.ArxivImportFromFile"/>	

	<bean id="DLIBDownloadPlugin" class="eu.dnetlib.download.plugin.DLibPlugin"/>	

	<bean id="HALPdfDocumentPlugin" class="eu.dnetlib.download.plugin.HALPdfDocumentPlugin"/>

	<bean id="HindawiDownloadPlugin" class="eu.dnetlib.download.plugin.HindawiDownloadPlugin"/>

	<bean id="DSpacePDFLinkPlugins" class="eu.dnetlib.download.plugin.DSpacePDFLinkPlugins"/>

</beans>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

	<modelVersion>4.0.0</modelVersion>

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

	</parent>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-download-plugins</artifactId>

	<version>2.1.17</version>

	<scm>

	  <developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-download-plugins/tags/dnet-download-plugins-2.1.17</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-download-service</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>org.jsoup</groupId>

			<artifactId>jsoup</artifactId>

			<version>1.7.2</version>

		</dependency>