D-Net

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-download-service/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-download-service"}

import java.security.Security;

import com.google.common.base.Function;

import eu.dnetlib.data.download.DownloadReportMap;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.worker.DownloadWorker;

import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord;

import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;

import eu.dnetlib.data.objectstore.rmi.Protocols;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.Mock;

import org.mockito.junit.MockitoJUnitRunner;

import static org.junit.Assert.assertEquals;

import static org.mockito.Matchers.any;

import static org.mockito.Mockito.when;

/**

 * Created by alessia on 22/12/16.

 */

@RunWith(MockitoJUnitRunner.class)

public class DownloadWorkerTest {

	private DownloadWorker dw;

	private DownloadReportMap reportMap;

	@Mock

	private ObjectStore objectStore;

	@Before

	public void prepare() throws Exception{

		//Security.insertProviderAt(new BouncyCastleProvider(),1);

		//System.setProperty("https.protocols", "TLSv1,TLSv1.1,TLSv1.2");

		reportMap = new DownloadReportMap();

		when(objectStore.feedObjectRecord(any(ObjectStoreRecord.class))).thenReturn("OK");

		dw = new DownloadWorker(null, objectStore, Protocols.valueOf("HTTP"), "application/pdf", 60000, 60000, 0,

				new Function<String, DownloadItem>() {

					@Override

					public DownloadItem apply(final String input) {

						return DownloadItem.newObjectfromJSON(input);

					}

				});

	}

	@Ignore

	@Test

	public void testSSL(){

		// String url = "https://econstor.eu/bitstream/10419/45606/1/658945068.pdf";

		String url = "https://www.preprints.org/manuscript/201909.0029/v1/download";

		dw.doDownload(1, reportMap, getDownloadItem("download.pdf", "download", url));

		System.out.println("Report map status "+reportMap.getStatus());

		assertEquals(1,reportMap.getTotalDownloaded());

	}

	private DownloadItem getDownloadItem(final String fileName, final String id, String url){

		DownloadItem item = new DownloadItem();

		item.setFileName(fileName);

		item.setIdItemMetadata(id);

		item.setOpenAccess("OPEN");

		item.setOriginalUrl(url);

		item.setUrl(url);

		return item;

	}

}

package eu.dnetlib.data.download;

import java.util.HashMap;

import com.google.gson.Gson;

/**

 * Created by sandro on 06/11/14.

 */

public class DownloadReportMap extends HashMap<String, DownloadReport> {

	/**

	 *

	 */

	private static final long serialVersionUID = -8881211500697708728L;

	private boolean status;

	private int totalDownloaded;

	public DownloadReportMap() {

		this.totalDownloaded = 0;

	}

	public void addDowload() {

		this.totalDownloaded++;

	}

	public boolean getStatus() {

		return status;

	}

	public void setStatus(final boolean status) {

		this.status = status;

	}

	public int getTotalDownloaded() {

		return totalDownloaded;

	}

	public void setTotalDownloaded(final int totalDownloaded) {

		this.totalDownloaded = totalDownloaded;

	}

	@Override

	public String toString() {

		Gson g = new Gson();

		return g.toJson(this);

	}

}

package eu.dnetlib.data.download.plugins;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import java.util.ArrayList;

import java.util.List;

public class DummyPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

    @Override

    public void setBasePath(String basePath) {

    }

    @Override

    public DownloadItem retrieveUrl(final DownloadItem item) {

        if (checkOpenAccess(item) == null) {

            return null;

        }

        if (filterByRegexp(item) == null) return null;

        String url = item.getUrl();

        if ((url == null) || (url.trim().length() == 0)) return null;

        @SuppressWarnings("unchecked")

        List<String> urls = new Gson().fromJson(url, ArrayList.class);

        if ((urls == null) || (urls.size() == 0)) {

            item.setOriginalUrl(null);

            item.setUrl(null);

            return item;

        }

        item.setOriginalUrl(urls.get(0));

        item.setUrl(urls.get(0));

        return item;

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> items) {

        return Iterables.transform(items, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    @Override

    public String getPluginName() {

        return "DummyPlugin";

    }

    @Override

    public String extractURL(String baseURL) {

        return null;

    }

}

package eu.dnetlib.data.download;

import eu.dnetlib.data.download.rmi.DownloadItem;

/**

 * Created by sandro on 06/11/14.

 */

public class DownloadReport {

    private int numberOfOccurrences;

    private String stackTrace;

	private DownloadItem downloadItem;

    public DownloadReport() {

        this.numberOfOccurrences =1;

    }

	public DownloadReport(int numberOfOccurrences, String stackTrace, DownloadItem downloadItem) {

		this.numberOfOccurrences = numberOfOccurrences;

        this.stackTrace = stackTrace;

		this.downloadItem = downloadItem;

	}

    public void incrementError(){

        this.numberOfOccurrences ++;

    }

    public void incrementError(int numberOfOccurrences){

        this.numberOfOccurrences += numberOfOccurrences;

    }

    /**

     * Getter for property 'numberOfOccurrences'.

     *

     * @return Value for property 'numberOfOccurrences'.

     */

    public int getNumberOfOccurrences() {

        return numberOfOccurrences;

    }

    /**

     * Setter for property 'numberOfOccurrences'.

     *

     * @param numberOfOccurrences Value to set for property 'numberOfOccurrences'.

     */

    public void setNumberOfOccurrences(int numberOfOccurrences) {

        this.numberOfOccurrences = numberOfOccurrences;

    }

    /**

     * Getter for property 'stackTrace'.

     *

     * @return Value for property 'stackTrace'.

     */

    public String getStackTrace() {

        return stackTrace;

    }

    /**

     * Setter for property 'stackTrace'.

     *

     * @param stackTrace Value to set for property 'stackTrace'.

     */

    public void setStackTrace(String stackTrace) {

        this.stackTrace = stackTrace;

    }

	public DownloadItem getDownloadItem() {

		return downloadItem;

	}

	public void setDownloadItem(final DownloadItem downloadItem) {

		this.downloadItem = downloadItem;

	}

}

package eu.dnetlib.data.download;

import java.util.List;

import javax.annotation.Resource;

import com.google.gson.Gson;

import com.google.gson.reflect.TypeToken;

import eu.dnetlib.data.download.rmi.DownloadServiceActions;

import eu.dnetlib.data.download.rmi.DownloadServiceException;

import eu.dnetlib.data.download.rmi.DownloadServiceFeeder;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.enabling.tools.blackboard.BlackboardServerAction;

import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;

import org.apache.commons.codec.binary.Base64;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

public class DownloadAction implements BlackboardServerAction<DownloadServiceActions> {

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(DownloadAction.class);

	private static String DEFAULT_NUMBER_OF_THREAD = "5";

	private static int DEFAULT_TIMEOUT_MS = 5000;

	@Resource

	private DownloadServiceFeeder downloadServiceFeeder;

	@Override

	public void execute(final BlackboardServerHandler handler, final BlackboardJob job) throws Exception {

		int connectTimeoutMs = DEFAULT_TIMEOUT_MS;

		int readTimeoutMs = DEFAULT_TIMEOUT_MS;

		int sleepTimeMs = 0;

		try {

			String epr = job.getParameters().get("epr");

			String plugin = job.getParameters().get("plugin");

			String objectStoreID = job.getParameters().get("objectStoreID");

			String protocol = job.getParameters().get("protocol");

			String mimeType = job.getParameters().get("mimeType");

			String numberOfThreads = job.getParameters().get("numberOfThreads");

			String basePath = job.getParameters().get("basePath");

			String regularExpression = job.getParameters().get("regularExpressions");

			String connectTimeoutMsStr = job.getParameters().get("connectTimeoutMs");

			String readTimeoutMsStr = job.getParameters().get("readTimeoutMs");

			String sleepTimeMsStr = job.getParameters().get("sleepTimeMs");

			log.debug("regular Expression: " + regularExpression);

			List<String> expressions = null;

			if (!StringUtils.isBlank(regularExpression)) {

				expressions = parseRegexList(regularExpression);

			}

			if (StringUtils.isBlank(numberOfThreads)) {

				log.warn("Cannot find numberOfThread, using default value " + DEFAULT_NUMBER_OF_THREAD);

				numberOfThreads = DEFAULT_NUMBER_OF_THREAD;

			}

			if (StringUtils.isNotBlank(connectTimeoutMsStr)) connectTimeoutMs = Integer.parseInt(connectTimeoutMsStr);

			if (StringUtils.isNotBlank(readTimeoutMsStr)) readTimeoutMs = Integer.parseInt(readTimeoutMsStr);

			if (StringUtils.isNotBlank(sleepTimeMsStr)) sleepTimeMs = Integer.parseInt(sleepTimeMsStr);

			log.info(String.format("downloading using plugin %s , protocol : %s into ObjectStore id %s  ", plugin, protocol, objectStoreID));

			handler.ongoing(job);

			DownloadReportMap response = downloadServiceFeeder.download(epr, plugin, objectStoreID, protocol, mimeType, Integer.parseInt(numberOfThreads),

					basePath, expressions, connectTimeoutMs, readTimeoutMs, sleepTimeMs);

			if (response.getStatus()) {

				log.info(String.format("Completed Download, plugin: %s, protocol: %s, objectStoreID: %s", plugin, protocol, objectStoreID));

				job.getParameters().put("total", "" + response.getTotalDownloaded());

				job.getParameters().put("report", Base64.encodeBase64String(response.toString().getBytes()));

				handler.done(job);

			} else {

				log.error("download response is false" + response);

				handler.failed(job, new DownloadServiceException(

						"oops! something bad happen to our download hamsters, downloaded so far: " + response.getTotalDownloaded()));

			}

		} catch (Exception e) {

			log.error("An error occur while starting download", e);

			handler.failed(job, new DownloadServiceException(e));

		}

	}

	private List<String> parseRegexList(final String regularExpression) {

		log.info("parsing regex list: " + regularExpression);

		return new Gson().fromJson(regularExpression, new TypeToken<List<String>>() {}.getType());

	}

}

package eu.dnetlib.data.download.rmi;

import java.util.List;

import java.util.concurrent.BlockingQueue;

import java.util.concurrent.ExecutorService;

import java.util.concurrent.Executors;

import java.util.concurrent.Future;

import javax.annotation.Resource;

import com.google.common.base.Joiner;

import com.google.common.collect.Lists;

import com.google.common.collect.Queues;

import eu.dnetlib.data.download.DownloadPluginEnumeratorImpl;

import eu.dnetlib.data.download.DownloadReport;

import eu.dnetlib.data.download.DownloadReportMap;

import eu.dnetlib.data.download.DownloadServiceImpl;

import eu.dnetlib.data.download.worker.DownloadWorker;

import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;

import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;

import eu.dnetlib.data.objectstore.rmi.Protocols;

import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

/**

 * The Class DownloadServiceFeeder.

 */

public class DownloadServiceFeeder {

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(DownloadServiceFeeder.class);

	private static final int MAX_NUM_FOUND = 10;

	/** The download plugin enumerator. */

	@Resource

	DownloadPluginEnumeratorImpl downloadPluginEnumerator;

	/** The result set client factory. */

	@Resource

	private ResultSetClientFactory resultSetClientFactory;

	/** The object store dao. */

	@Autowired

	private ObjectStoreDao objectStoreDao;

	public static void reportException(final DownloadReportMap report, final DownloadItem di, final Throwable e) {

		final String className = e.getClass().getName();

		if (!report.containsKey(className)) {

			final DownloadReport dr = new DownloadReport();

			dr.setStackTrace(Joiner.on("\tat ").join(e.getStackTrace()));

			if (di != null) {

				dr.setDownloadItem(di);

			}

			report.put(className, dr);

		} else {

			report.get(className).incrementError();

		}

	}

	/**

	 * Download and feed file into the objectStore .

	 *

	 * @param epr

	 *            the end-point reference of the result-set of Serialized DownloadItem

	 * @param plugin

	 *            the plugin used to retrieve the correct URL

	 * @param objectStoreID

	 *            the object store id to store the data

	 * @param protocol

	 *            the protocol used to download the file

	 * @param mimeType

	 *            the mime type of the Files

	 * @param numberOfThreads

	 *            the number of threads to use for download at the same time

	 * @throws DownloadServiceException

	 *             the download service exception

	 * @throws ObjectStoreServiceException

	 */

	public DownloadReportMap download(final String epr,

			final String plugin,

			final String objectStoreID,

			final String protocol,

			final String mimeType,

			final int numberOfThreads,

			final String basePath,

			final List<String> regularExpression,

			final int connectTimeoutMs,

			final int readTimeoutMs, final int sleepTimeMs) throws DownloadServiceException, ObjectStoreServiceException {

		final DownloadPlugin downloadPlugin = downloadPluginEnumerator.get(plugin);

		if ((basePath != null) && (basePath.isEmpty() == false)) {

			downloadPlugin.setBasePath(basePath);

		}

		final Iterable<String> urlInfo = resultSetClientFactory.getClient(epr);

		final BlockingQueue<String> itemsQueue = Queues.newArrayBlockingQueue(1024);

		final ExecutorService executor = Executors.newFixedThreadPool(numberOfThreads);

		final ObjectStore objStore = objectStoreDao.getObjectStore(objectStoreID);

		final List<Future<DownloadReportMap>> responses = Lists.newArrayList();

		final DownloadReportMap pluginReport = new DownloadReportMap();

		pluginReport.setStatus(true);

		if (regularExpression != null) {

			downloadPlugin.setRegularExpression(regularExpression);

		}

		for (int i = 0; i < numberOfThreads; i++) {

			responses.add(executor

					.submit(new DownloadWorker(itemsQueue, objStore, Protocols.valueOf(protocol), mimeType, connectTimeoutMs, readTimeoutMs, sleepTimeMs,

							input -> {

								if (input == null) {

									log.debug("Input is null");

									return null;

								}

								if (input.equals(DownloadServiceImpl.END_QUEUE_STRING)) return DownloadServiceImpl.END_QUEUE;

								DownloadItem di = null;

								try {

									di = DownloadItem.newObjectfromJSON(input);

									if (downloadPlugin.retrieveUrl(di) == null) {

										di.setUrl(null);

										di.setOriginalUrl(null);

									}

									return di;

								} catch (Throwable e) {

									reportException(pluginReport, di, e);

									log.debug("Exception on transform item :" + input, e);

									return null;

								}

							})));

		}

		int i = 0;

		int null_found = 0;

		if (urlInfo != null) {

			for (final String downloadItem : urlInfo) {

				if (downloadItem != null) {

					null_found = 0;

					if ((i++ % 1000) == 0) {

						log.debug("Read " + i);

					}

					try {

						itemsQueue.put(downloadItem);

					} catch (final Exception e) {

						log.error("An error occurred while populating the download items queue: " + Joiner.on("\tat ").join(e.getStackTrace()));

					}

				} else {

					if (null_found++ > MAX_NUM_FOUND) {

						break;

					}

				}

			}

		}

		try {

			itemsQueue.put(DownloadServiceImpl.END_QUEUE_STRING);

		} catch (final InterruptedException e) {

			log.error("An error occurred adding the loop terminator: " + Joiner.on("\tat ").join(e.getStackTrace()));

		}

		final DownloadReportMap resultMap = getDownloadReportMap(responses, pluginReport);

		executor.shutdown();

		return resultMap;

	}

	private DownloadReportMap getDownloadReportMap(final List<Future<DownloadReportMap>> responses, final DownloadReportMap pluginReport) {

		final DownloadReportMap resultMap = new DownloadReportMap();

		resultMap.setStatus(true);

		for (final Future<DownloadReportMap> currentResponse : responses) {

			try {

				final DownloadReportMap currentMap = currentResponse.get();

				mergeReport(resultMap, currentMap);

				log.info("Status " + currentMap.getStatus());

				resultMap.setStatus(resultMap.getStatus() && currentMap.getStatus());

				resultMap.setTotalDownloaded(currentMap.getTotalDownloaded() + resultMap.getTotalDownloaded());

			} catch (final Exception e) {

				log.error(e);

				resultMap.setStatus(false);

			}

		}

		mergeReport(resultMap, pluginReport);

		return resultMap;

	}

	private void mergeReport(final DownloadReportMap resultMap, final DownloadReportMap currentMap) {

		for (final String key : currentMap.keySet()) {

			if (!resultMap.containsKey(key)) {

				resultMap.put(key, currentMap.get(key));

			} else {

				final DownloadReport currentReport = currentMap.get(key);

				resultMap.get(key).incrementError(currentReport.getNumberOfOccurrences());

			}

			resultMap.setTotalDownloaded(resultMap.getTotalDownloaded() + currentMap.getTotalDownloaded());

			resultMap.setStatus(resultMap.getStatus() & currentMap.getStatus());

		}

	}

}

package eu.dnetlib.data.download.rmi;

public enum DownloadServiceActions {

	DOWNLOAD,

}

package eu.dnetlib.data.download.worker;

import java.io.IOException;

import java.net.HttpURLConnection;

import java.net.URL;

import java.net.URLConnection;

import java.util.concurrent.BlockingQueue;

import java.util.concurrent.Callable;

import java.util.concurrent.TimeUnit;

import com.google.common.base.Function;

import com.google.common.base.Joiner;

import eu.dnetlib.data.download.DownloadReportMap;

import eu.dnetlib.data.download.DownloadServiceImpl;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadServiceFeeder;

import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord;

import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;

import eu.dnetlib.data.objectstore.rmi.Protocols;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * The Class DownloadWorker is a worker responsible to download the data into the object store.

 */

public class DownloadWorker implements Callable<DownloadReportMap> {

	/**

	 * The Constant log.

	 */

	private static final Log log = LogFactory.getLog(DownloadWorker.class);

	private static final int MAX_NULLS = 5 ;

	/**

	 * The queue.

	 */

	private BlockingQueue<String> queue = null;

	/**

	 * The object store.

	 */

	private ObjectStore objectStore = null;

	/**

	 * The protocol.

	 */

	private Protocols protocol;

	/**

	 * The mime type.

	 */

	private String mimeType;

	private Function<String, DownloadItem> converter;

	private int connectTimeoutMs;

	private int readTimeoutMs;

	/**

	 * ms to wait between two subsequent download request

	 **/

	private int sleepTimeMs;

	private String userAgent = "Mozilla/5.0 (compatible; OAI; +http://www.openaire.eu)";  // "OpenAIRE aggregation system/2.2.5 ; (OpenAccess aggregator; mailto:helpdesk@openaire.eu)"

	/**

	 * Instantiates a new download worker.

	 *

	 * @param queue

	 *            the queue

	 * @param objectStore

	 *            the object store

	 * @param protocol

	 *            the protocol

	 * @param mimeType

	 *            the mime type

	 */

	public DownloadWorker(final BlockingQueue<String> queue,

			final ObjectStore objectStore,

			final Protocols protocol,

			final String mimeType,

			final int connectTimeoutMs,

			final int readTimeoutMs, final int sleepTimeMs,

			final Function<String, DownloadItem> converter) {

		this.setConverter(converter);

		this.setQueue(queue);

		this.setObjectStore(objectStore);

		this.setMimeType(mimeType);

		this.setProtocol(protocol);

		this.setConnectTimeoutMs(connectTimeoutMs);

		this.setReadTimeoutMs(readTimeoutMs);

		this.setSleepTimeMs(sleepTimeMs);

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see java.lang.Runnable#run()

	 */

	@Override

	public DownloadReportMap call() throws Exception {

		final DownloadReportMap report = new DownloadReportMap();

		final long threadId = Thread.currentThread().getId();

		int nullCounter = 0;

		try {

			String takedObject = queue.poll(5, TimeUnit.SECONDS);

			while (!DownloadServiceImpl.END_QUEUE_STRING.equals(takedObject) && nullCounter < MAX_NULLS) {

				if (takedObject == null) {

					nullCounter++;

				}

				if (log.isDebugEnabled()) {

					log.debug(takedObject);

				}

				final DownloadItem di = getConverter().apply(takedObject);

				log.debug(threadId + ": Reading " + takedObject);

				if (di == null) {

					log.debug("the current download item is Null, skipping");

					DownloadServiceFeeder.reportException(report, null, new IllegalArgumentException("found null DownloadItem"));

				} else if (StringUtils.isNotBlank(di.getUrl()) && !checkIfExists(di)) {

					doDownload(threadId, report, di);

				}

				takedObject = queue.poll(5, TimeUnit.SECONDS);

				log.debug(String.format("%s: next object from queue %s, remaining items: %s", threadId, takedObject, queue.size()));

			}

		} catch (final Exception e) {

			log.debug("An error occured : " + Joiner.on("\tat ").join(e.getStackTrace()));

			DownloadServiceFeeder.reportException(report, null, e);

			report.setStatus(false);

			return report;

		} finally {

			log.info(String.format("%s: finalising queue, remaining items: %s, nulls: %s", threadId, queue.size(), nullCounter));

			final boolean res = queue.offer(DownloadServiceImpl.END_QUEUE_STRING, 5, TimeUnit.SECONDS);

			log.info("put terminator in queue: " + res);

		}

		log.info("CLOSED THREAD " + threadId);

		report.setStatus(true);

		return report;

	}

	public void doDownload(final long threadId, final DownloadReportMap report, final DownloadItem di) {

		try {

			if (getSleepTimeMs() > 0) {

				log.debug(threadId + ": I will sleep for " + getSleepTimeMs() + " ms, as requested...");

				Thread.sleep(getSleepTimeMs());

			}

			final URL toDownload = followURL(threadId, new URL(di.getUrl()), report, di);

			final ObjectStoreRecord record = new ObjectStoreRecord();

			final ObjectStoreFile metadata = new ObjectStoreFile();

			metadata.setObjectID(di.getFileName());

			metadata.setMetadataRelatedID(di.getIdItemMetadata());

			metadata.setAccessProtocol(this.protocol);

			metadata.setMimeType(this.mimeType);

			metadata.setDownloadedURL(di.getOriginalUrl());

			record.setFileMetadata(metadata);

			log.debug(threadId + ": opening connection " + toDownload);

			final URLConnection connection = toDownload.openConnection();

			// set user-agent

			connection.setRequestProperty("User-Agent", userAgent);

			connection.setConnectTimeout(getConnectTimeoutMs());

			connection.setReadTimeout(getReadTimeoutMs());

			//connection.setRequestProperty("User-Agent", "OpenAIRE aggregation system/2.2.5 ; (OpenAccess aggregator; mailto:helpdesk@openaire.eu) ");

			log.debug(threadId + ": getting input stream from " + toDownload);

			record.setInputStream(connection.getInputStream());

			log.debug(threadId + ": feeding object from  " + toDownload + " into objectstore ...");

			objectStore.feedObjectRecord(record);

			report.addDowload();

			log.debug(threadId + ": saved object " + metadata.toJSON());

		} catch (final Throwable e) {

			log.debug(threadId + ": error downloading Item: " + di.toJSON(), e);

			DownloadServiceFeeder.reportException(report, di, e);

		}

	}

	private URL followURL(final long threadId, final URL inputURL, final DownloadReportMap report, final DownloadItem di) throws IOException {

		final String ptrcl = inputURL.getProtocol();

		if (ptrcl.startsWith("file")) {

			log.debug("the protocol is File, returning " + inputURL);

			return inputURL;

		}

		HttpURLConnection conn = (HttpURLConnection) inputURL.openConnection();

		conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manually.

		conn.setReadTimeout(getReadTimeoutMs());

		conn.setConnectTimeout(getConnectTimeoutMs());

		// set user-agent

		conn.setRequestProperty("User-Agent", userAgent);

		String location = inputURL.toString();

		log.debug(threadId + " : followURL connecting  " + inputURL);

		conn.connect();

		log.debug(threadId + " : followURL connected  " + inputURL);

		int responseCode = conn.getResponseCode();

		log.debug(threadId + " : followURL " + inputURL + ", response code: " + responseCode);

		if ((responseCode >= 300) && (responseCode < 400)) {

			location = conn.getHeaderFields().get("Location").get(0);

			conn.disconnect();

			log.debug(threadId + " : followURL disconnected  " + inputURL);

		}

		if (!location.equals(inputURL.toString())) return new URL(location);

		return inputURL;

	}

	private boolean checkIfExists(final DownloadItem di) {

		try {

			return objectStore.deliverObject(di.getFileName()) != null;

		} catch (final ObjectStoreServiceException e) {

			log.debug(e.getMessage());

			return false;

		}

	}

	/**

	 * Sets the object store.

	 *

	 * @param objectStore

	 *            the objectStore to set

	 */

	public void setObjectStore(final ObjectStore objectStore) {

		this.objectStore = objectStore;

	}

	/**

	 * Sets the queue.

	 *

	 * @param queue

	 *            the queue to set

	 */

	public void setQueue(final BlockingQueue<String> queue) {

		this.queue = queue;

	}

	/**

	 * Sets the protocol.

	 *

	 * @param protocol

	 *            the protocol to set

	 */

	public void setProtocol(final Protocols protocol) {

		this.protocol = protocol;

	}

	/**

	 * Sets the mime type.

	 *

	 * @param mimeType

	 *            the mimeType to set

	 */

	public void setMimeType(final String mimeType) {

		this.mimeType = mimeType;

	}

	public Function<String, DownloadItem> getConverter() {

		return converter;

	}

	public void setConverter(final Function<String, DownloadItem> converter) {

		this.converter = converter;

	}

	public int getReadTimeoutMs() {

		return readTimeoutMs;

	}

	public void setReadTimeoutMs(final int readTimeoutMs) {

		this.readTimeoutMs = readTimeoutMs;

	}

	public int getConnectTimeoutMs() {

		return connectTimeoutMs;

	}

	public void setConnectTimeoutMs(final int connectTimeoutMs) {

		this.connectTimeoutMs = connectTimeoutMs;

	}

	public int getSleepTimeMs() {

		return sleepTimeMs;

	}

	public void setSleepTimeMs(final int sleepTimeMs) {

		this.sleepTimeMs = sleepTimeMs;

	}

}

package eu.dnetlib.data.download;

import java.util.Map;

import org.springframework.beans.BeansException;

import org.springframework.beans.factory.BeanFactory;

import org.springframework.beans.factory.ListableBeanFactory;

import com.google.common.collect.Maps;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginEnumerator;

import eu.dnetlib.data.download.rmi.DownloadServiceException;

/**

 * The Class DownloadPluginEnumeratorImpl.

 */

public class DownloadPluginEnumeratorImpl implements DownloadPluginEnumerator {

	/**

	 * bean factory.

	 */

	private ListableBeanFactory beanFactory;

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadPluginEnumerator#getAll()

	 */

	@Override

	public Map<String, DownloadPlugin> getAll() {

		return beanFactory.getBeansOfType(DownloadPlugin.class);

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadPluginEnumerator#getByProtocols()

	 */

	@Override

	public Map<String, DownloadPlugin> getByProtocols() {

		final Map<String, DownloadPlugin> res = Maps.newHashMap();

		for (DownloadPlugin cp : getAll().values()) {

			res.put(cp.getPluginName().toLowerCase(), cp);

		}

		return res;

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see org.springframework.beans.factory.BeanFactoryAware#setBeanFactory(org.springframework.beans.factory.BeanFactory)

	 */

	@Override

	public void setBeanFactory(final BeanFactory beanFactory) throws BeansException {

		this.beanFactory = (ListableBeanFactory) beanFactory;

	}

	/**

	 * Gets the bean factory.

	 *

	 * @return the bean factory

	 */

	public ListableBeanFactory getBeanFactory() {

		return beanFactory;

	}

	/**

	 * Get given DownloadPlugin or throws exception.

	 *

	 * @param protocol

	 *            the protocol

	 * @return the download plugin

	 * @throws eu.dnetlib.data.download.rmi.DownloadServiceException

	 *             the download service exception

	 */

	public DownloadPlugin get(final String protocol) throws DownloadServiceException {

		final DownloadPlugin plugin = getByProtocols().get(protocol.toLowerCase());

		if (plugin == null) throw new DownloadServiceException("plugin not found for name: " + protocol);

		return plugin;

	}

}

package eu.dnetlib.data.download;

import java.security.KeyManagementException;

import java.security.NoSuchAlgorithmException;

import java.security.Security;

import java.security.cert.CertificateException;

import java.util.ArrayList;

import java.util.List;

import javax.annotation.Resource;

import javax.net.ssl.*;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadService;

import eu.dnetlib.data.download.rmi.DownloadServiceException;

import eu.dnetlib.data.download.rmi.DownloadServiceFeeder;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;

import eu.dnetlib.enabling.tools.AbstractBaseService;

import eu.dnetlib.enabling.tools.blackboard.NotificationHandler;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import org.springframework.beans.factory.annotation.Value;

/**

 * The Class DownloadServiceImpl.

 */

public class DownloadServiceImpl extends AbstractBaseService implements DownloadService {

	/**

	 * The Constant END_QUEUE.

	 */

	public static final DownloadItem END_QUEUE = new DownloadItem();

	public static final String END_QUEUE_STRING = "END_DOWNLOAD";

	/**

	 * The Constant log.

	 */

	private static final Log log = LogFactory.getLog(DownloadServiceImpl.class);

	private static int DEFAULT_CONNECT_TIMEOUT_MS = 5000;

	private static int DEFAULT_READ_TIMEOUT_MS = 5000;

	/**

	 * The download plugin enumerator.

	 */

	@Resource

	private DownloadPluginEnumeratorImpl downloadPluginEnumerator;

	@Resource

	private DownloadServiceFeeder downloadfeeder;

	/**

	 * The notification handler.

	 */

	private NotificationHandler notificationHandler;

	@Value("${services.download.http.sslcheck}")

	private boolean checkCerts;

	@Value("${services.download.https.protocols}")

	private String httpsProtocols;

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadService#downloadFromResultSet(javax.xml.ws.wsaddressing.W3CEndpointReference,

	 * java.lang.String, java.lang.String, java.lang.String, java.lang.String)

	 */

	@Override

	public void downloadFromResultSet(final W3CEndpointReference resultSet,

			final String plugin,

			final String objectStoreID,

			final String protocol,

			final String mimeType) throws DownloadServiceException {

		log.info(String.format("download using plugin '%s' , protocol '%s' into ObjectStore '%s'", plugin, protocol, objectStoreID));

		try {

			downloadfeeder.download(resultSet.toString(), plugin, objectStoreID, protocol, mimeType, 5, null, null, DEFAULT_CONNECT_TIMEOUT_MS,

					DEFAULT_READ_TIMEOUT_MS, 0);

		} catch (ObjectStoreServiceException e) {

			log.error(e);

		}

		log.info(String.format("download completed using plugin '%s' , protocol '%s' into ObjectStore '%s'", plugin, protocol, objectStoreID));

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.enabling.tools.AbstractBaseService#notify(java.lang.String, java.lang.String, java.lang.String, java.lang.String)

	 */

	@Override

	public void notify(final String subscriptionId, final String topic, final String isId, final String message) {

		getNotificationHandler().notified(subscriptionId, topic, isId, message);

	}

	@Override

	public void start() {

		if (!checkCerts) {

			log.info("disabling SSL check ...");

			new Thread(new Runnable() {

				@Override

				public void run() {

					disableSslVerification();

				}

			}).start();

		}

		//TODO: we should remove references to BouncyCastle. We are not expecting issues like #2520 with Java8

		//Security.insertProviderAt(new BouncyCastleProvider(),1);

		//System.setProperty("https.protocols", httpsProtocols);

	}

	private void disableSslVerification() {

		try {

			// Create a trust manager that does not validate certificate chains

			TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() {

				@Override

				public void checkClientTrusted(final java.security.cert.X509Certificate[] x509Certificates, final String s) throws CertificateException {

				}

				@Override

				public void checkServerTrusted(final java.security.cert.X509Certificate[] x509Certificates, final String s) throws CertificateException {

				}

				@Override

				public java.security.cert.X509Certificate[] getAcceptedIssuers() {

					return null;

				}

			} };

			// Install the all-trusting trust manager

			SSLContext sc = SSLContext.getInstance("SSL");

			sc.init(null, trustAllCerts, new java.security.SecureRandom());

			HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());

			// Create all-trusting host name verifier

			HostnameVerifier allHostsValid = new HostnameVerifier() {

				@Override

				public boolean verify(final String hostname, final SSLSession session) {

					return true;

				}

			};

			// Install the all-trusting host verifier

			HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid);

			log.info("disabled SSL check");

		} catch (NoSuchAlgorithmException e) {

			log.error(e.getMessage(), e);

		} catch (KeyManagementException e) {

			log.error(e.getMessage(), e);

		}

	}

	/**

	 * @return the downloadfeeder

	 */

	public DownloadServiceFeeder getDownloadfeeder() {

		return downloadfeeder;

	}

	/**

	 * @param downloadfeeder the downloadfeeder to set

	 */

	public void setDownloadfeeder(final DownloadServiceFeeder downloadfeeder) {

		this.downloadfeeder = downloadfeeder;

	}

	/**

	 * @return the notificationHandler

	 */

	public NotificationHandler getNotificationHandler() {

		return notificationHandler;

	}

	/**

	 * @param notificationHandler the notificationHandler to set

	 */

	@Required

	public void setNotificationHandler(final NotificationHandler notificationHandler) {

		this.notificationHandler = notificationHandler;

	}

	@Override

	public List<String> listPlugins() throws DownloadServiceException {

		List<String> result = new ArrayList<String>();

		result.addAll(downloadPluginEnumerator.getAll().keySet());

		return result;

	}

}

services.download.http.sslcheck=false

services.download.https.protocols=TLSv1,TLSv1.1,TLSv1.2

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws"

	xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing"

	xmlns:p="http://www.springframework.org/schema/p" xmlns:http="http://cxf.apache.org/transports/http/configuration"

	xmlns:t="http://dnetlib.eu/springbeans/t" xmlns:template="http://dnetlib.eu/springbeans/template"

	xmlns:util="http://www.springframework.org/schema/util"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd

                                    http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd

                                    http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd

                                    http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd

                            http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd

                            http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd

                            http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">

    <bean id="downloadNotificationHandler"

		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerExecutorNotificationHandler"

		p:blackboardExecutor-ref="downloadBlackboardExecutor" />

	<bean id = "DownloadServiceFeeder" class ="eu.dnetlib.data.download.rmi.DownloadServiceFeeder" />

	<bean id="downloadBlackboardExecutor"

		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerActionExecutor"

		p:blackboardHandler-ref="blackboardHandler"

		p:actionType="eu.dnetlib.data.download.rmi.DownloadServiceActions"

		p:incomplete="false">

		<property name="actionMap">

			<map>

				<entry key="DOWNLOAD">

					<bean class="eu.dnetlib.data.download.DownloadAction"/>

				</entry>				

			</map>

		</property>

	</bean>		

    <bean id="DummyPlugin" class="eu.dnetlib.data.download.plugins.DummyPlugin" />

	<bean id="downloadSerive" class="eu.dnetlib.data.download.DownloadServiceImpl"

		init-method="start" p:notificationHandler-ref="downloadNotificationHandler" />

	<bean id="downloadPluginEnumerator" class="eu.dnetlib.data.download.DownloadPluginEnumeratorImpl" />

	<jaxws:endpoint id="downloadServiceEndpoint"

		implementor="#downloadSerive" implementorClass="eu.dnetlib.data.download.rmi.DownloadService"

		address="/downloadService" />

	<template:instance name="serviceRegistrationManager"

		t:serviceRegistrationManagerClass="eu.dnetlib.enabling.tools.registration.ValidatingServiceRegistrationManagerImpl"

		t:name="downloadServiceRegistrationManager" t:service="downloadSerive"

		t:endpoint="downloadServiceEndpoint" t:jobScheduler="jobScheduler"

		t:serviceRegistrator="blackboardServiceRegistrator" />

</beans>

log4j.rootLogger=WARN, CONSOLE

log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender

log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout

log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n

org.apache.cxf.Logger=org.apache.cxf.common.logging.Log4jLogger

### Application Level ###

log4j.logger.eu.dnetlib=INFO

log4j.logger.eu.dnetlib.data.download=DEBUG

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-download-service</artifactId>

	<version>2.2.6</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-download-service/tags/dnet-download-service-2.2.6</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-download-service-rmi</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-service-common</artifactId>

			<version>[2.1.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-modular-objectstore-service</artifactId>

			<version>[5.0.0,6.0.0)</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.mockito</groupId>

			<artifactId>mockito-core</artifactId>

			<version>${mockito.version}</version>

			<scope>test</scope>

		</dependency>

	</dependencies>

</project>