D-Net

package eu.dnetlib.download.plugin;

import java.io.IOException;

import java.net.HttpURLConnection;

import java.net.URL;

import java.util.ArrayList;

import java.util.List;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

public class DSpacePDFLinkPlugins extends AbstractDownloadPlugin implements DownloadPlugin {

    /**

     * The Constant log.

     */

    private static final Log log = LogFactory.getLog(DSpacePDFLinkPlugins.class);

    private final static int maxNumberJump = 10;

    private final static int maxNumberConnectRetries = 5;

    /**

     * Milliseconds used to backoff in case of connection errors.

     */

    private final static int BACKOFF_FACTOR = 100;

    private String getHTTPRedirectedURL(final String mainURL) throws Exception {

        URL startURL = new URL(mainURL);

        HttpURLConnection conn = (HttpURLConnection) startURL.openConnection();

        conn.setConnectTimeout(AbstractDownloadPlugin.DEFAULT_TIMEOUT);

        conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manually.

        HttpURLConnection.setFollowRedirects(true);

        String location = mainURL;

        int numJump = 1;

        int responseCode = conn.getResponseCode();

        while ((responseCode >= 300) && (responseCode < 400) && (numJump++ < maxNumberJump)) {

            location = conn.getHeaderFields().get("Location").get(0);

            conn.disconnect();

            startURL = new URL(location);

            conn = (HttpURLConnection) startURL.openConnection();

            conn.setConnectTimeout(AbstractDownloadPlugin.DEFAULT_TIMEOUT);

            conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manually.

            HttpURLConnection.setFollowRedirects(true);

            responseCode = conn.getResponseCode();

        }

        conn.disconnect();

        if (!((responseCode >= 200) && (responseCode < 300)))

            return null;

        return location;

    }

    /**

     * Extract url.

     *

     * @param url the url

     * @return the string

     */

    @Override

    public String extractURL(final String url) throws DownloadPluginException {

        try {

            final String location = getHTTPRedirectedURL(url);

            if (location == null) {

                return null;

            }

            Document doc = null;

            int retries = 0;

            boolean success = false;

            while(retries < maxNumberConnectRetries) {

                try {

                    doc = Jsoup.connect(location).timeout(AbstractDownloadPlugin.DEFAULT_TIMEOUT).get();

                    success = true;

                    break;

                } catch (IOException e) {

                    final int millis = BACKOFF_FACTOR * (retries + 1);

                    log.debug(String.format("backoff for %s ms before retrying on %s", millis, location));

                    Thread.sleep(millis);

                }

                retries++;

            }

            if (!success) {

                throw new DownloadPluginException("reached max number of connect retries for URL: " + location);

            }

            final Elements links = doc.select("meta[content][name=citation_pdf_url]");

            for (Element link : links) {

                String linkValue = link.attr("content");

                if (regularExpression != null) {

                    for (String regex : regularExpression) {

                        if (linkValue.matches(regex)) {

                            return linkValue;

                        }

                    }

                } else {

                    //if(linkValue.matches("^http.*pdf$")){

                        return linkValue;

                    //}

                }

            }

            return null;

        } catch (Throwable e) {

	        throw new DownloadPluginException("Error on extract URL", e);

        }

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, input -> retrieveUrl(input));

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "DSpacePDFLinkPlugins";

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        if (checkOpenAccess(input) == null) return null;

        String url = input.getOriginalUrl();

        if ((url == null) || (url.trim().length() == 0)) return input;

        @SuppressWarnings("unchecked")

        List<String> urls = new Gson().fromJson(url, ArrayList.class);

        if ((urls == null) || (urls.size() == 0)) return input;

        if (checkUrlsNotNull(input, urls))

            return input;

        input.setOriginalUrl(null);

        input.setUrl(null);

        return input;

    }

    @Override

    public void setBasePath(final String basePath) {

        // TODO Auto-generated method stub

    }

}