D-Net

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

	<modelVersion>4.0.0</modelVersion>

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet-parent</artifactId>

		<version>1.0.0</version>

	</parent>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-download-plugins</artifactId>

	<version>2.1.16-SNAPSHOT</version>

	<scm>

	  <developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-download-plugins/trunk</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-download-service</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>org.jsoup</groupId>

			<artifactId>jsoup</artifactId>

			<version>1.7.2</version>

		</dependency>

		<dependency>

			<groupId>joda-time</groupId>

			<artifactId>joda-time</artifactId>

			<version>2.3</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

	</dependencies>

</project>

package eu.dnetlib.download.plugin;

import java.nio.file.FileSystems;

import java.nio.file.Files;

import java.nio.file.Path;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

/**

 * Created by sandro on 3/1/16.

 */

public class HindawiDownloadPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

	private String basePath;

	private Gson gson;

	public HindawiDownloadPlugin() {

		this.gson = new Gson();

	}

	@Override

	public String extractURL(final String baseURL) throws DownloadPluginException {

		return null;

	}

	@Override

	public void setBasePath(final String basePath) {

		this.basePath = basePath;

	}

	/*

	 * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

	@Override

	public String getPluginName() {

		return "HindawiDownloadPlugin";

	}

	@Override

	public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) throws DownloadPluginException {

		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

			@Override

			public DownloadItem apply(final DownloadItem input) {

				return retrieveUrl(input);

			}

		});

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

	 */

	@Override

	public DownloadItem retrieveUrl(final DownloadItem input) throws DownloadPluginException {

		List<String> inputList = this.gson.fromJson(input.getUrl(), ArrayList.class);

		if (inputList == null || inputList.size() == 0) {

			input.setUrl(null);

			return input;

		}

		for (String fileUrl : inputList) {

			Path inputhPath = FileSystems.getDefault().getPath(basePath).resolve("." + fileUrl);

			if (Files.exists(inputhPath)) {

				input.setOriginalUrl("file://" + inputhPath.toString());

				input.setUrl("file://" + inputhPath.toString());

				return input;

			}

		}

		input.setUrl(null);

		return input;

	}

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

// TODO: Auto-generated Javadoc

/**

 * The Class EuropePMC.

 */

public class EuropePMC extends AbstractDownloadPlugin implements DownloadPlugin {

    private static final Log log = LogFactory.getLog(EuropePMC.class);

    /**

     * The base path.

     */

    private String basePath;

    // //*[local-name()='metadata']//*[local-name()='identifier' and ./@identifierType='pmc']/text()

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "europePMCDownloadPlugin";

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) throws DownloadPluginException {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) throws DownloadPluginException {

        try {

            String url = input.getOriginalUrl();

            if ((url == null) || (url.trim().length() == 0)) return input;

            @SuppressWarnings("unchecked")

            List<String> urls = new Gson().fromJson(url, ArrayList.class);

            log.debug(String.format("urls is %s", url));

            if ((urls == null) || (urls.size() == 0)) return input;

            input.setFileName(input.getIdItemMetadata());

            Boolean added = false;

            for (String s : urls) {

                if (s.startsWith("http")) {

                    log.debug(String.format("found url starting with http replace original URL with %s", s));

                    input.setOriginalUrl(s);

                } else if (s.startsWith("PMC")) {

                    String correctUrl = s.replace("PMC", "");

                    log.debug(String.format("found url starting with PMC %s", correctUrl));

                    added = true;

                    String path = PathRetreiver.getInstance(basePath).getPathForPMCID(Integer.parseInt(correctUrl));

                    if (path != null) {

                        input.setUrl("file://" + path);

                    } else {

                        input.setUrl(path);

                    }

                }

            }

            if (added == false) {

                input.setOriginalUrl(null);

                input.setUrl(null);

            }

            return input;

        } catch (Throwable e) {

            log.error("Exception on Download Plugin");

            log.error(e);

            throw new DownloadPluginException(e);

        }

    }

    /**

     * Gets the base path.

     *

     * @return the basePath

     */

    public String getBasePath() {

        return basePath;

    }

    /**

     * Sets the base path.

     *

     * @param basePath the basePath to set

     */

    @Override

    public void setBasePath(final String basePath) {

        this.basePath = basePath;

    }

    @Override

    public String extractURL(String baseURL) throws DownloadPluginException {

        return null;

    }

}

package eu.dnetlib.download.plugin;

import java.net.HttpURLConnection;

import java.net.URL;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

public class FollowPDFLinkPlugins extends AbstractDownloadPlugin implements DownloadPlugin {

    /**

     * The Constant log.

     */

    private static final Log log = LogFactory.getLog(FollowPDFLinkPlugins.class);

    private final static int maxNumberJump = 10;

    /**

     * Extract url.

     *

     * @param url the url

     * @return the string

     */

    @Override

    public String extractURL(final String url) throws DownloadPluginException {

        try {

            URL startURL = new URL(url);

            HttpURLConnection conn = (HttpURLConnection) startURL.openConnection();

	        conn.setConnectTimeout(AbstractDownloadPlugin.DEFAULT_TIMEOUT);

            conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manully.

            HttpURLConnection.setFollowRedirects(true);

            String location = url;

            int numJump =1;

            int responseCode = conn.getResponseCode();

            while ((responseCode >= 300) && (responseCode < 400) &&  (numJump++ < maxNumberJump)) {

                location = conn.getHeaderFields().get("Location").get(0);

                conn.disconnect();

                startURL = new URL(location);

                conn = (HttpURLConnection) startURL.openConnection();

                conn.setConnectTimeout(AbstractDownloadPlugin.DEFAULT_TIMEOUT);

                conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manully.

                HttpURLConnection.setFollowRedirects(true);

                responseCode = conn.getResponseCode();

            }

            conn.disconnect();

            if (!((responseCode >= 200) && (responseCode < 300)))

                return null;

            Document doc = Jsoup.connect(location).get();

            Elements links = doc.select("a[href$=.pdf]");

            for (Element link : links) {

                String linkValue = link.attr("abs:href");

                if (regularExpression != null) {

                    for (String regex : regularExpression) {

                        if (linkValue.matches(regex))

                            return linkValue;

                    }

                } else

                    return linkValue;

            }

            return null;

        } catch (Throwable e) {

	        throw new DownloadPluginException("Error on extract URL", e);

        }

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "FollowPDFLinkPlugins";

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        if (checkOpenAccess(input) == null) return null;

        String url = input.getOriginalUrl();

        if ((url == null) || (url.trim().length() == 0)) return input;

        @SuppressWarnings("unchecked")

        List<String> urls = new Gson().fromJson(url, ArrayList.class);

        if ((urls == null) || (urls.size() == 0)) return input;

        if (checkUrlsNotNull(input, urls))

            return input;

        input.setOriginalUrl(null);

        input.setUrl(null);

        return input;

    }

    @Override

    public void setBasePath(final String basePath) {

        // TODO Auto-generated method stub

    }

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

/**

 * The Class ELisDownloadPlugin.

 */

public class ELisDownloadPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

    /**

     * The Constant log.

     */

    private static final Log log = LogFactory.getLog(ELisDownloadPlugin.class);

    /**

     * Extract url.

     *

     * @param url the url

     * @return the string

     */

    @Override

    public String extractURL(final String url) throws DownloadPluginException {

        try {

            Document doc = Jsoup.connect(url).get();

            Elements links = doc.select("a[href$=.pdf]");

            for (Element link : links) {

                String linkvalue = link.attr("abs:href");

                if (!linkvalue.toLowerCase().contains("thumbnailversion")) {

                    return linkvalue;

                }

            }

            return null;

        } catch (Exception e) {

            throw new DownloadPluginException("Error on extract URL", e);

        }

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "ELisDownloadPlugin";

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        if (checkOpenAccess(input) == null) {

            return null;

        }

        String url = input.getOriginalUrl();

        if (url == null || url.trim().length() == 0) {

            return input;

        }

        @SuppressWarnings("unchecked")

        List<String> urls = new Gson().fromJson(url, ArrayList.class);

        if (urls == null || urls.size() == 0) {

            return input;

        }

        if (checkUrlsNotNull(input, urls))

            return input;

        input.setOriginalUrl(null);

        input.setUrl(null);

        return input;

    }

    @Override

    public void setBasePath(final String basePath) {

        // TODO Auto-generated method stub

    }

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

// TODO: Auto-generated Javadoc

/**

 * The Class EuropePMC.

 */

public class EuropePMC extends AbstractDownloadPlugin implements DownloadPlugin {

    private static final Log log = LogFactory.getLog(EuropePMC.class);

    /**

     * The base path.

     */

    private String basePath;

    // //*[local-name()='metadata']//*[local-name()='identifier' and ./@identifierType='pmc']/text()

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "europePMCDownloadPlugin";

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) throws DownloadPluginException {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) throws DownloadPluginException {

        try {

            String url = input.getOriginalUrl();

            if ((url == null) || (url.trim().length() == 0)) return input;

            @SuppressWarnings("unchecked")

            List<String> urls = new Gson().fromJson(url, ArrayList.class);

            log.debug(String.format("urls is %s", url));

            if ((urls == null) || (urls.size() == 0)) return input;

            input.setFileName(input.getIdItemMetadata());

            Boolean added = false;

            for (String s : urls) {

                if (s.startsWith("http")) {

                    log.debug(String.format("found url starting with http replace original URL with %s", s));

                    input.setOriginalUrl(s);

                } else if (s.startsWith("PMC")) {

                    String correctUrl = s.replace("PMC", "");

                    log.debug(String.format("found url starting with PMC %s", correctUrl));

                    added = true;

                    String path = PathRetreiver.getInstance(basePath).getPathForPMCID(Integer.parseInt(correctUrl));

                    if (path != null) {

                        input.setUrl("file://" + path);

                    } else {

                        input.setUrl(path);

                    }

                }

            }

            if (added == false) {

                input.setOriginalUrl(null);

                input.setUrl(null);

            }

            return input;

        } catch (Throwable e) {

            log.error("Exception on Download Plugin");

            log.error(e);

            throw new DownloadPluginException(e);

        }

    }

    /**

     * Gets the base path.

     *

     * @return the basePath

     */

    public String getBasePath() {

        return basePath;

    }

    /**

     * Sets the base path.

     *

     * @param basePath the basePath to set

     */

    @Override

    public void setBasePath(final String basePath) {

        this.basePath = basePath;

    }

    @Override

    public String extractURL(String baseURL) throws DownloadPluginException {

        return null;

    }

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

public class ArxivDownloadPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        if (checkOpenAccess(input) == null) {

            return null;

        }

        String url = input.getOriginalUrl();

        if (url == null || url.trim().length() == 0) {

            return input;

        }

        @SuppressWarnings("unchecked")

        List<String> urls = new Gson().fromJson(url, ArrayList.class);

        if (urls == null || urls.size() == 0) {

            return input;

        }

        for (String s : urls) {

            if (s.startsWith("http")) {

                input.setOriginalUrl(s);

                String correctUrl = s.replace("abs", "pdf");

                correctUrl += ".pdf";

                input.setUrl(correctUrl);

            }

        }

        return input;

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    @Override

    public String getPluginName() {

        return "ArxivDownloadPlugin";

    }

    @Override

    public void setBasePath(final String basePath) {

    }

    @Override

    public String extractURL(String baseURL) throws DownloadPluginException {

        return null;

    }

}

/**

 *

 */

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * @author sandro

 */

public class HALPdfDocumentPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

    private static final Log log = LogFactory.getLog(HALPdfDocumentPlugin.class); // NOPMD by marko on 11/24/08 5:02 PM

    /**

     * {@inheritDoc}

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        // TODO Auto-generated method stub

        return "HALPdfDocumentPlugin";

    }

    /**

     * {@inheritDoc}

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrls(java.lang.Iterable)

     */

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /**

     * {@inheritDoc}

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        try {

            if (checkOpenAccess(input) == null) return null;

            if (input == null) return null;

            String url = input.getOriginalUrl();

            if ((url == null) || (url.trim().length() == 0)) return input;

            @SuppressWarnings("unchecked")

            List<String> urls = new Gson().fromJson(url, ArrayList.class);

            if ((urls == null) || (urls.size() == 0)) return input;

            if (checkUrlsNotNull(input, urls)) return input;

            input.setOriginalUrl(null);

            input.setUrl(null);

            return input;

        } catch (Throwable e) {

            log.error("Error on retreiving URL", e);

            input.setOriginalUrl(null);

            input.setUrl(null);

            return input;

        }

    }

    /**

     * {@inheritDoc}

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#setBasePath(java.lang.String)

     */

    @Override

    public void setBasePath(final String arg0) {

        // TODO Auto-generated method stub

    }

    @Override

    public String extractURL(String baseURL) throws DownloadPluginException {

        return baseURL.toLowerCase().endsWith("document") ? baseURL : null;

    }

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

/**

 * The Class EasyPDFDownloadPlugin.

 */

public class EasyPDFDownloadPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        if (checkOpenAccess(input) == null) {

            return null;

        }

        if (input == null) {

            return null;

        }

        String url = input.getOriginalUrl();

        if (url == null || url.trim().length() == 0) {

            return input;

        }

        @SuppressWarnings("unchecked")

        List<String> urls = new Gson().fromJson(url, ArrayList.class);

        if (urls == null || urls.size() == 0) {

            return input;

        }

        if (checkUrlsNotNull(input, urls))

            return input;

        input.setOriginalUrl(null);

        input.setUrl(null);

        return input;

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "easyPDFDownloadPlugin";

    }

    @Override

    public void setBasePath(final String basePath) {

        // TODO Auto-generated method stub

    }

    @Override

    public String extractURL(String baseURL) throws DownloadPluginException {

        return baseURL.endsWith(".pdf") ? baseURL : null;

    }

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

/**

 * The Class ELisDownloadPlugin.

 */

public class ELisDownloadPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

    /**

     * The Constant log.

     */

    private static final Log log = LogFactory.getLog(ELisDownloadPlugin.class);

    /**

     * Extract url.

     *

     * @param url the url

     * @return the string

     */

    @Override

    public String extractURL(final String url) throws DownloadPluginException {

        try {

            Document doc = Jsoup.connect(url).get();

            Elements links = doc.select("a[href$=.pdf]");

            for (Element link : links) {

                String linkvalue = link.attr("abs:href");

                if (!linkvalue.toLowerCase().contains("thumbnailversion")) {

                    return linkvalue;

                }

            }

            return null;

        } catch (Exception e) {

            throw new DownloadPluginException("Error on extract URL", e);

        }

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "ELisDownloadPlugin";

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        if (checkOpenAccess(input) == null) {

            return null;

        }

        String url = input.getOriginalUrl();

        if (url == null || url.trim().length() == 0) {

            return input;

        }

        @SuppressWarnings("unchecked")

        List<String> urls = new Gson().fromJson(url, ArrayList.class);

        if (urls == null || urls.size() == 0) {

            return input;

        }

        if (checkUrlsNotNull(input, urls))

            return input;

        input.setOriginalUrl(null);

        input.setUrl(null);

        return input;

    }

    @Override

    public void setBasePath(final String basePath) {

        // TODO Auto-generated method stub

    }

}

/**

 *

 */

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * @author sandro

 */

public class HALPdfDocumentPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

    private static final Log log = LogFactory.getLog(HALPdfDocumentPlugin.class); // NOPMD by marko on 11/24/08 5:02 PM

    /**

     * {@inheritDoc}

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        // TODO Auto-generated method stub

        return "HALPdfDocumentPlugin";

    }

    /**

     * {@inheritDoc}

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrls(java.lang.Iterable)

     */

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /**

     * {@inheritDoc}

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        try {

            if (checkOpenAccess(input) == null) return null;

            if (input == null) return null;

            String url = input.getOriginalUrl();

            if ((url == null) || (url.trim().length() == 0)) return input;

            @SuppressWarnings("unchecked")

            List<String> urls = new Gson().fromJson(url, ArrayList.class);

            if ((urls == null) || (urls.size() == 0)) return input;

            if (checkUrlsNotNull(input, urls)) return input;

            input.setOriginalUrl(null);

            input.setUrl(null);

            return input;

        } catch (Throwable e) {

            log.error("Error on retreiving URL", e);

            input.setOriginalUrl(null);

            input.setUrl(null);

            return input;

        }

    }

    /**

     * {@inheritDoc}

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#setBasePath(java.lang.String)

     */

    @Override

    public void setBasePath(final String arg0) {

        // TODO Auto-generated method stub

    }

    @Override

    public String extractURL(String baseURL) throws DownloadPluginException {

        return baseURL.toLowerCase().endsWith("document") ? baseURL : null;

    }

}

package eu.dnetlib.download.plugin;

/**

 * The Class InfoPath.

 */

public class InfoPath {

	/** The lower. */

	private int lower;

	/** The upper. */

	private int upper;

	/** The path. */

	private String path;

	/**

	 * Gets the lower.

	 *

	 * @return the lower

	 */

	public int getLower() {

		return lower;

	}

	/**

	 * Sets the lower.

	 *

	 * @param lower

	 *            the new lower

	 */

	public void setLower(final int lower) {

		this.lower = lower;

	}

	/**

	 * Gets the upper.

	 *

	 * @return the upper

	 */

	public int getUpper() {

		return upper;

	}

	/**

	 * Sets the upper.

	 *

	 * @param upper

	 *            the new upper

	 */

	public void setUpper(final int upper) {

		this.upper = upper;

	}

	/**

	 * Gets the path.

	 *

	 * @return the path

	 */

	public String getPath() {

		return path;

	}

	/**

	 * Sets the path.

	 *

	 * @param path

	 *            the new path

	 */

	public void setPath(final String path) {

		this.path = path;

	}

}

package eu.dnetlib.download.plugin;

import junit.framework.Assert;

import org.junit.Before;

import org.junit.Test;

import com.google.gson.Gson;

import com.google.gson.GsonBuilder;

import eu.dnetlib.data.download.rmi.DownloadItem;

public class ArxivImportFromFileTest {

	@Before

	public void setUp() throws Exception {}

	@Test

	public void testREgEx() {

		String regExp = "\\d{4}\\.\\d{4}";

		String input = "1308.0021";

		Assert.assertTrue(input.matches(regExp));

	}

	@Test

	public void testPlugin() {

		DownloadItem item = new DownloadItem();

		item.setOpenAccess("OPEN");

		String[] myList = new String[] { "http://arxiv.org/abs/1308.0001", "b" };

		Gson g = new GsonBuilder().disableHtmlEscaping().create();

		item.setUrl(g.toJson(myList));

		ArxivImportFromFile plugin = new ArxivImportFromFile();

		plugin.setBasePath("/test/path");

		plugin.retrieveUrl(item);

		Assert.assertNotNull(item);

	}

}

package eu.dnetlib.download.plugin;

import junit.framework.Assert;

import org.junit.Before;

import org.junit.Test;

import com.google.gson.Gson;

import com.google.gson.GsonBuilder;

import eu.dnetlib.data.download.rmi.DownloadItem;

public class ArxivImportFromFileTest {

	@Before

	public void setUp() throws Exception {}

	@Test

	public void testREgEx() {

		String regExp = "\\d{4}\\.\\d{4}";

		String input = "1308.0021";

		Assert.assertTrue(input.matches(regExp));

	}

	@Test

	public void testPlugin() {

		DownloadItem item = new DownloadItem();

		item.setOpenAccess("OPEN");

		String[] myList = new String[] { "http://arxiv.org/abs/1308.0001", "b" };

		Gson g = new GsonBuilder().disableHtmlEscaping().create();

		item.setUrl(g.toJson(myList));

		ArxivImportFromFile plugin = new ArxivImportFromFile();

		plugin.setBasePath("/test/path");

		plugin.retrieveUrl(item);

		Assert.assertNotNull(item);

	}

}

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-download-plugins/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "dnet-download-plugins"}

package eu.dnetlib.download.plugin;

import java.net.HttpURLConnection;

import java.net.URL;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

public class FollowPDFLinkPlugins extends AbstractDownloadPlugin implements DownloadPlugin {

    /**

     * The Constant log.

     */

    private static final Log log = LogFactory.getLog(FollowPDFLinkPlugins.class);

    private final static int maxNumberJump = 10;

    /**

     * Extract url.

     *

     * @param url the url

     * @return the string

     */

    @Override

    public String extractURL(final String url) throws DownloadPluginException {

        try {

            URL startURL = new URL(url);

            HttpURLConnection conn = (HttpURLConnection) startURL.openConnection();

	        conn.setConnectTimeout(AbstractDownloadPlugin.DEFAULT_TIMEOUT);

            conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manully.

            HttpURLConnection.setFollowRedirects(true);

            String location = url;

            int numJump =1;

            int responseCode = conn.getResponseCode();

            while ((responseCode >= 300) && (responseCode < 400) &&  (numJump++ < maxNumberJump)) {

                location = conn.getHeaderFields().get("Location").get(0);

                conn.disconnect();

                startURL = new URL(location);

                conn = (HttpURLConnection) startURL.openConnection();

                conn.setConnectTimeout(AbstractDownloadPlugin.DEFAULT_TIMEOUT);

                conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manully.

                HttpURLConnection.setFollowRedirects(true);

                responseCode = conn.getResponseCode();

            }

            conn.disconnect();

            if (!((responseCode >= 200) && (responseCode < 300)))

                return null;

            Document doc = Jsoup.connect(location).get();

            Elements links = doc.select("a[href$=.pdf]");

            for (Element link : links) {

                String linkValue = link.attr("abs:href");

                if (regularExpression != null) {

                    for (String regex : regularExpression) {

                        if (linkValue.matches(regex))

                            return linkValue;

                    }

                } else

                    return linkValue;

            }

            return null;

        } catch (Throwable e) {

	        throw new DownloadPluginException("Error on extract URL", e);

        }

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "FollowPDFLinkPlugins";

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        if (checkOpenAccess(input) == null) return null;

        String url = input.getOriginalUrl();

        if ((url == null) || (url.trim().length() == 0)) return input;

        @SuppressWarnings("unchecked")

        List<String> urls = new Gson().fromJson(url, ArrayList.class);

        if ((urls == null) || (urls.size() == 0)) return input;

        if (checkUrlsNotNull(input, urls))

            return input;

        input.setOriginalUrl(null);

        input.setUrl(null);

        return input;

    }

    @Override

    public void setBasePath(final String basePath) {

        // TODO Auto-generated method stub

    }

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

// TODO: Auto-generated Javadoc

/**

 * The Class EuropePMC.

 */

public class EuropePMC extends AbstractDownloadPlugin implements DownloadPlugin {

    private static final Log log = LogFactory.getLog(EuropePMC.class);

    /**

     * The base path.

     */

    private String basePath;

    // //*[local-name()='metadata']//*[local-name()='identifier' and ./@identifierType='pmc']/text()

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "europePMCDownloadPlugin";

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) throws DownloadPluginException {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /*

     * (non-Javadoc)

     *