D-Net

package eu.dnetlib.download.plugin;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

import java.net.HttpURLConnection;

import java.net.URL;

import java.util.ArrayList;

import java.util.List;

public class DSpacePDFLinkPlugins extends AbstractDownloadPlugin implements DownloadPlugin {

    /**

     * The Constant log.

     */

    private static final Log log = LogFactory.getLog(DSpacePDFLinkPlugins.class);

    private final static int maxNumberJump = 10;

    private String getHTTPRedirectedURL(final String mainURL) throws Exception {

        URL startURL = new URL(mainURL);

        HttpURLConnection conn = (HttpURLConnection) startURL.openConnection();

        conn.setConnectTimeout(AbstractDownloadPlugin.DEFAULT_TIMEOUT);

        conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manully.

        HttpURLConnection.setFollowRedirects(true);

        String location = mainURL;

        int numJump =1;

        int responseCode = conn.getResponseCode();

        while ((responseCode >= 300) && (responseCode < 400) &&  (numJump++ < maxNumberJump)) {

            location = conn.getHeaderFields().get("Location").get(0);

            conn.disconnect();

            startURL = new URL(location);

            conn = (HttpURLConnection) startURL.openConnection();

            conn.setConnectTimeout(AbstractDownloadPlugin.DEFAULT_TIMEOUT);

            conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manully.

            HttpURLConnection.setFollowRedirects(true);

            responseCode = conn.getResponseCode();

        }

        conn.disconnect();

        if (!((responseCode >= 200) && (responseCode < 300)))

            return null;

        return location;

    }

    /**

     * Extract url.

     *

     * @param url the url

     * @return the string

     */

    @Override

    public String extractURL(final String url) throws DownloadPluginException {

        try {

            final String location = getHTTPRedirectedURL(url);

            if (location == null)

                return null;

            Document doc = Jsoup.connect(location).get();

            Elements links = doc.select("meta[content$=.pdf]");

            for (Element link : links) {

                String linkValue = link.attr("content");

                if (regularExpression != null) {

                    for (String regex : regularExpression) {

                        if (linkValue.matches(regex))

                            return linkValue;

                    }

                } else

                    return linkValue;

            }

            return null;

        } catch (Throwable e) {

	        throw new DownloadPluginException("Error on extract URL", e);

        }

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, input -> retrieveUrl(input));

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "DSpacePDFLinkPlugins";

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        if (checkOpenAccess(input) == null) return null;

        String url = input.getOriginalUrl();

        if ((url == null) || (url.trim().length() == 0)) return input;

        @SuppressWarnings("unchecked")

        List<String> urls = new Gson().fromJson(url, ArrayList.class);

        if ((urls == null) || (urls.size() == 0)) return input;

        if (checkUrlsNotNull(input, urls))

            return input;

        input.setOriginalUrl(null);

        input.setUrl(null);

        return input;

    }

    @Override

    public void setBasePath(final String basePath) {

        // TODO Auto-generated method stub

    }

}