D-Net

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-download-service/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "dnet-download-service"}

package eu.dnetlib.data.download;

import java.util.HashMap;

import com.google.gson.Gson;

/**

 * Created by sandro on 06/11/14.

 */

public class DownloadReportMap extends HashMap<String, DownloadReport> {

	/**

	 *

	 */

	private static final long serialVersionUID = -8881211500697708728L;

	private boolean status;

	private int totalDownloaded;

	public DownloadReportMap() {

		this.totalDownloaded = 0;

	}

	public void addDowload() {

		this.totalDownloaded++;

	}

	public boolean getStatus() {

		return status;

	}

	public void setStatus(final boolean status) {

		this.status = status;

	}

	public int getTotalDownloaded() {

		return totalDownloaded;

	}

	public void setTotalDownloaded(final int totalDownloaded) {

		this.totalDownloaded = totalDownloaded;

	}

	@Override

	public String toString() {

		Gson g = new Gson();

		return g.toJson(this);

	}

}

package eu.dnetlib.data.download.plugins;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

public class DummyPlugin implements DownloadPlugin {

	@Override

	public DownloadItem retrieveUrl(final DownloadItem item) {

		String url = item.getUrl();

		if ((url == null) || (url.trim().length() == 0)) return null;

		@SuppressWarnings("unchecked")

		List<String> urls = new Gson().fromJson(url, ArrayList.class);

		if ((urls == null) || (urls.size() == 0)) {

			item.setOriginalUrl(null);

			item.setUrl(null);

			return item;

		}

		item.setOriginalUrl(urls.get(0));

		item.setUrl(urls.get(0));

		return item;

	}

	@Override

	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> items) {

		return Iterables.transform(items, new Function<DownloadItem, DownloadItem>() {

			@Override

			public DownloadItem apply(final DownloadItem input) {

				return retrieveUrl(input);

			}

		});

	}

	@Override

	public String getPluginName() {

		return "DummyPlugin";

	}

	@Override

	public void setBasePath(final String basePath) {

		// TODO Auto-generated method stub

	}

	/**

	 * {@inheritDoc}

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getRegularExpression()

	 */

	@Override

	public List<String> getRegularExpression() {

		// TODO Auto-generated method stub

		return null;

	}

	/**

	 * {@inheritDoc}

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#setRegularExpression(java.util.List)

	 */

	@Override

	public void setRegularExpression(final List<String> regularExpression) {

		// TODO Auto-generated method stub

	}

}

package eu.dnetlib.data.download;

/**

 * Created by sandro on 06/11/14.

 */

public class DownloadReport {

    private int numberOfOccurrences;

    private String stackTrace;

    private String recordId;

    public DownloadReport() {

        this.numberOfOccurrences =1;

    }

    public DownloadReport(int numberOfOccurrences, String stackTrace, String recordId) {

        this.numberOfOccurrences = numberOfOccurrences;

        this.stackTrace = stackTrace;

        this.recordId = recordId;

    }

    public void incrementError(){

        this.numberOfOccurrences ++;

    }

    public void incrementError(int numberOfOccurrences){

        this.numberOfOccurrences += numberOfOccurrences;

    }

    /**

     * Getter for property 'numberOfOccurrences'.

     *

     * @return Value for property 'numberOfOccurrences'.

     */

    public int getNumberOfOccurrences() {

        return numberOfOccurrences;

    }

    /**

     * Setter for property 'numberOfOccurrences'.

     *

     * @param numberOfOccurrences Value to set for property 'numberOfOccurrences'.

     */

    public void setNumberOfOccurrences(int numberOfOccurrences) {

        this.numberOfOccurrences = numberOfOccurrences;

    }

    /**

     * Getter for property 'stackTrace'.

     *

     * @return Value for property 'stackTrace'.

     */

    public String getStackTrace() {

        return stackTrace;

    }

    /**

     * Setter for property 'stackTrace'.

     *

     * @param stackTrace Value to set for property 'stackTrace'.

     */

    public void setStackTrace(String stackTrace) {

        this.stackTrace = stackTrace;

    }

    /**

     * Getter for property 'recordId'.

     *

     * @return Value for property 'recordId'.

     */

    public String getRecordId() {

        return recordId;

    }

    /**

     * Setter for property 'recordId'.

     *

     * @param recordId Value to set for property 'recordId'.

     */

    public void setRecordId(String recordId) {

        this.recordId = recordId;

    }

}

package eu.dnetlib.data.download;

import java.lang.reflect.Type;

import java.util.List;

import javax.annotation.Resource;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.google.common.reflect.TypeToken;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.DownloadServiceActions;

import eu.dnetlib.data.download.rmi.DownloadServiceException;

import eu.dnetlib.data.download.rmi.DownloadServiceFeeder;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.enabling.tools.blackboard.BlackboardServerAction;

import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;

public class DownloadAction implements BlackboardServerAction<DownloadServiceActions> {

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(DownloadAction.class);

	@Resource

	DownloadServiceFeeder downloadServiceFeeder;

	private static String DEFAULT_NUMBER_OF_THREAD = "5";

	private Type typeToken = new TypeToken<List<String>>() {

		/**

		 *

		 */

		private static final long serialVersionUID = 5683238626707746063L;

	}.getType();

	@Override

	public void execute(final BlackboardServerHandler handler, final BlackboardJob job) throws Exception {

		try {

			String epr = job.getParameters().get("epr");

			String plugin = job.getParameters().get("plugin");

			String objectStoreID = job.getParameters().get("objectStoreID");

			String protocol = job.getParameters().get("protocol");

			String mimeType = job.getParameters().get("mimeType");

			String numberOfThreads = job.getParameters().get("thread");

			String basePath = job.getParameters().get("basePath");

			String regularExpression = job.getParameters().get("regularExpressions");

			log.debug("regular Expression: "+regularExpression);

			List<String> expressions = null;

			if (!StringUtils.isBlank(regularExpression)) {

				expressions =new Gson().fromJson(regularExpression, typeToken);

			}

			if ((numberOfThreads == "") || (numberOfThreads == null)) {

				log.warn("Cannot find numberOfThread, using default value " + DEFAULT_NUMBER_OF_THREAD);

				numberOfThreads = DEFAULT_NUMBER_OF_THREAD;

			}

			log.info(String.format("downloading using plugin %s , protocol : %s into ObjectStore id %s  ", plugin, protocol, objectStoreID));

			handler.ongoing(job);

			DownloadReportMap response = downloadServiceFeeder.download(epr, plugin, objectStoreID, protocol, mimeType, Integer.parseInt(numberOfThreads),

					basePath, expressions);

			job.getParameters().put("downloadReport", "" + response.getTotalDownloaded());

			if (response.getStatus()) {

				log.info(String.format("Completed Download", plugin, protocol, objectStoreID));

				handler.done(job);

			} else {

				log.error("response is false" + response);

				handler.failed(job, new DownloadServiceException("Opps something bad happen to our download hamsters" + response.toString()));

			}

		} catch (Exception e) {

			log.error("An error occur while starting download",e);

			handler.failed(job, new DownloadServiceException(e));

		}

	}

}

package eu.dnetlib.data.download.rmi;

import java.util.List;

import java.util.concurrent.ArrayBlockingQueue;

import java.util.concurrent.BlockingQueue;

import java.util.concurrent.ExecutorService;

import java.util.concurrent.Executors;

import java.util.concurrent.Future;

import javax.annotation.Resource;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import com.google.common.base.Function;

import com.google.common.base.Joiner;

import com.google.common.collect.Lists;

import eu.dnetlib.data.download.DownloadPluginEnumeratorImpl;

import eu.dnetlib.data.download.DownloadReport;

import eu.dnetlib.data.download.DownloadReportMap;

import eu.dnetlib.data.download.DownloadServiceImpl;

import eu.dnetlib.data.download.worker.DownloadWorker;

import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;

import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;

import eu.dnetlib.data.objectstore.rmi.Protocols;

import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;

/**

 * The Class DownloadServiceFeeder.

 */

public class DownloadServiceFeeder {

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(DownloadServiceFeeder.class);

	/** The download plugin enumerator. */

	@Resource

	DownloadPluginEnumeratorImpl downloadPluginEnumerator;

	/** The result set client factory. */

	@Resource

	private ResultSetClientFactory resultSetClientFactory;

	/** The object store dao. */

	@Autowired

	private ObjectStoreDao objectStoreDao;

	/**

	 * Download and feed file into the objectStore .

	 *

	 * @param epr

	 *            the end-point reference of the result-set of Serialized DownloadItem

	 * @param plugin

	 *            the plugin used to retrieve the correct URL

	 * @param objectStoreID

	 *            the object store id to store the data

	 * @param protocol

	 *            the protocol used to download the file

	 * @param mimeType

	 *            the mime type of the Files

	 * @param numberOfThreads

	 *            the number of threads to use for download at the same time

	 * @throws DownloadServiceException

	 *             the download service exception

	 * @throws ObjectStoreServiceException

	 */

	public DownloadReportMap download(final String epr,

			final String plugin,

			final String objectStoreID,

			final String protocol,

			final String mimeType,

			final int numberOfThreads,

			final String basePath,

			final List<String> regularExpression

			) throws DownloadServiceException, ObjectStoreServiceException {

		final DownloadPlugin downloadPlugin = downloadPluginEnumerator.get(plugin);

		if ((basePath != null) && (basePath.isEmpty() == false)) {

			downloadPlugin.setBasePath(basePath);

		}

		final Iterable<String> urlInfo = resultSetClientFactory.getClient(epr);

		final BlockingQueue<String> itemsQueue = new ArrayBlockingQueue<String>(1024);

		final ExecutorService executor = Executors.newFixedThreadPool(numberOfThreads);

		final ObjectStore objStore = objectStoreDao.getObjectStore(objectStoreID);

		final List<Future<DownloadReportMap>> responses = Lists.newArrayList();

		if (regularExpression!= null) {

			downloadPlugin.setRegularExpression(regularExpression);

		}

		for (int i = 0; i < numberOfThreads; i++) {

			responses.add(executor.submit(new DownloadWorker(itemsQueue, objStore, Protocols.valueOf(protocol), mimeType, new Function<String, DownloadItem>() {

				@Override

				public DownloadItem apply(final String input) {

					if (input == null) {

						log.error("Input is null");

						return null;

					}

					if (input.equals(DownloadServiceImpl.END_QUEUE_STRING)) return DownloadServiceImpl.END_QUEUE;

					try {

						final DownloadItem di = DownloadItem.newObjectfromJSON(input);

						if (downloadPlugin.retrieveUrl(di) == null) {

							di.setUrl(null);

							di.setOriginalUrl(null);

						}

						return di;

					} catch (Exception e) {

						log.error("Exception on trasform item :"+input,e);

						return null;

					}

				}

			})));

		}

		int i = 0;

		if(urlInfo!= null) {

			for (final String downloadItem : urlInfo) {

				if (downloadItem != null) {

					if ((i++ % 1000) == 0) {

						log.debug("Read " + i);

					}

					try {

						itemsQueue.put(downloadItem);

					} catch (final Exception e) {

						log.error("An error occurred while populating the download items queue: " + Joiner.on("\tat ").join(e.getStackTrace()));

					}

				}

			}

		}

		try {

			itemsQueue.put(DownloadServiceImpl.END_QUEUE_STRING);

		} catch (final InterruptedException e) {

			log.error("An error occurred adding the loop terminator: " + Joiner.on("\tat ").join(e.getStackTrace()));

		}

		final DownloadReportMap resultMap = new DownloadReportMap();

		resultMap.setStatus(true);

		for (final Future<DownloadReportMap> currentResponses : responses) {

			try {

				final DownloadReportMap currentMap = currentResponses.get();

				for (final String key : currentMap.keySet()) {

					if (!resultMap.containsKey(key)) {

						resultMap.put(key, currentMap.get(key));

						resultMap.setTotalDownloaded(currentMap.getTotalDownloaded());

						resultMap.setStatus(currentMap.getStatus());

					} else {

						final DownloadReport currentReport = currentMap.get(key);

						resultMap.get(key).incrementError(currentReport.getNumberOfOccurrences());

					}

				}

				log.info("Status " + currentMap.getStatus());

				resultMap.setStatus(resultMap.getStatus() && currentMap.getStatus());

				resultMap.setTotalDownloaded(currentMap.getTotalDownloaded() + resultMap.getTotalDownloaded());

			} catch (final Exception e) {

				log.error(e);

				resultMap.setStatus(false);

			}

		}

		executor.shutdown();

		return resultMap;

	}

}

package eu.dnetlib.data.download.rmi;

public enum DownloadServiceActions {

	DOWNLOAD,

}

package eu.dnetlib.data.download.worker;

import java.io.IOException;

import java.net.HttpURLConnection;

import java.net.URL;

import java.util.concurrent.BlockingQueue;

import java.util.concurrent.Callable;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.google.common.base.Function;

import com.google.common.base.Joiner;

import eu.dnetlib.data.download.DownloadReport;

import eu.dnetlib.data.download.DownloadReportMap;

import eu.dnetlib.data.download.DownloadServiceImpl;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord;

import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;

import eu.dnetlib.data.objectstore.rmi.Protocols;

/**

 * The Class DownloadWorker is a worker responsible to download the data into the object store.

 */

public class DownloadWorker implements Callable<DownloadReportMap> {

	/**

	 * The Constant log.

	 */

	private static final Log log = LogFactory.getLog(DownloadWorker.class);

	/**

	 * The queue.

	 */

	private BlockingQueue<String> queue = null;

	/**

	 * The object store.

	 */

	private ObjectStore objectStore = null;

	/**

	 * The protocol.

	 */

	private Protocols protocol;

	/**

	 * The mime type.

	 */

	private String mimeType;

	private Function<String, DownloadItem> converter;

	/**

	 * Instantiates a new download worker.

	 *

	 * @param queue

	 *            the queue

	 * @param objectStore

	 *            the object store

	 * @param protocol

	 *            the protocol

	 * @param mimeType

	 *            the mime type

	 */

	public DownloadWorker(final BlockingQueue<String> queue, final ObjectStore objectStore, final Protocols protocol, final String mimeType,

			final Function<String, DownloadItem> converter) {

		this.setConverter(converter);

		this.setQueue(queue);

		this.setObjectStore(objectStore);

		this.setMimeType(mimeType);

		this.setProtocol(protocol);

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see java.lang.Runnable#run()

	 */

	@Override

	public DownloadReportMap call() throws Exception {

		final DownloadReportMap report = new DownloadReportMap();

		try {

			String takedObject = queue.take();

			DownloadItem di = getConverter().apply(takedObject);

			if (log.isDebugEnabled()) {

				log.debug(takedObject);

			}

			while (DownloadServiceImpl.END_QUEUE != di) {

				log.debug(Thread.currentThread().getId()+": Reading  "+takedObject);

				if (di == null) {

					log.info("the current download item is Null, skipping");

				}

				if ((di != null) && (di.getUrl() != null) && (di.getUrl().length() != 0) && !checkIfExists(di)) {

					try {

						final URL toDownload = followURL(new URL(di.getUrl()), report, di);

						final ObjectStoreRecord record = new ObjectStoreRecord();

						final ObjectStoreFile metadata = new ObjectStoreFile();

						metadata.setObjectID(di.getFileName());

						metadata.setMetadataRelatedID(di.getIdItemMetadata());

						metadata.setAccessProtocol(this.protocol);

						metadata.setMimeType(this.mimeType);

						metadata.setDownloadedURL(di.getOriginalUrl());

						record.setFileMetadata(metadata);

						record.setInputStream(toDownload.openStream());

						objectStore.feedObjectRecord(record);

						report.addDowload();

						log.debug("Saved object " + metadata.toJSON());

					} catch (final Throwable e) {

						log.error("An error occur inside the loop: ",e);

						// reportException(report, di, e);

					}

				}

				takedObject = queue.take();

				log.debug(Thread.currentThread().getId()+": Next Object From Queue  "+takedObject);

				di = getConverter().apply(takedObject);

				if (log.isDebugEnabled()) {

					log.debug(takedObject);

				}

			}

			queue.put(DownloadServiceImpl.END_QUEUE_STRING);

		} catch (final Exception e) {

			log.error("An error occured : " + Joiner.on("\tat ").join(e.getStackTrace()));

			report.setStatus(false);

			return report;

		}

		log.info("CLOSED THREAD " + Thread.currentThread().getId());

		report.setStatus(true);

		return report;

	}

	private void reportException(final DownloadReportMap report, final DownloadItem di, final Exception e) {

		final String className = e.getClass().getName();

		if (!report.containsKey(className)) {

			final DownloadReport dr = new DownloadReport();

			dr.setStackTrace(Joiner.on("\tat ").join(e.getStackTrace()));

			dr.setRecordId(di.getIdItemMetadata());

			report.put(className, dr);

		} else {

			report.get(className).incrementError();

		}

	}

	private URL followURL(final URL inputURL, final DownloadReportMap report, final DownloadItem di) throws IOException {

		final String ptrcl = inputURL.getProtocol();

		if (ptrcl.startsWith("file")) {

			log.debug("the protocol is File, returning " + inputURL);

			return inputURL;

		}

		HttpURLConnection conn;

		conn = (HttpURLConnection) inputURL.openConnection();

		conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manully.

		HttpURLConnection.setFollowRedirects(true);

		String location = inputURL.toString();

		if ((conn.getResponseCode() >= 300) && (conn.getResponseCode() < 400)) {

			location = conn.getHeaderFields().get("Location").get(0);

			conn.disconnect();

		}

		if (!location.equals(inputURL.toString())) return new URL(location);

		return inputURL;

	}

	private boolean checkIfExists(final DownloadItem di) {

		try {

			return objectStore.deliverObject(di.getFileName()) != null;

		} catch (final ObjectStoreServiceException e) {

			log.debug(e.getMessage());

			return false;

		}

	}

	/**

	 * Sets the object store.

	 *

	 * @param objectStore

	 *            the objectStore to set

	 */

	public void setObjectStore(final ObjectStore objectStore) {

		this.objectStore = objectStore;

	}

	/**

	 * Sets the queue.

	 *

	 * @param queue

	 *            the queue to set

	 */

	public void setQueue(final BlockingQueue<String> queue) {

		this.queue = queue;

	}

	/**

	 * Sets the protocol.

	 *

	 * @param protocol

	 *            the protocol to set

	 */

	public void setProtocol(final Protocols protocol) {

		this.protocol = protocol;

	}

	/**

	 * Sets the mime type.

	 *

	 * @param mimeType

	 *            the mimeType to set

	 */

	public void setMimeType(final String mimeType) {

		this.mimeType = mimeType;

	}

	public Function<String, DownloadItem> getConverter() {

		return converter;

	}

	public void setConverter(final Function<String, DownloadItem> converter) {

		this.converter = converter;

	}

}

package eu.dnetlib.data.download;

import java.util.Map;

import org.springframework.beans.BeansException;

import org.springframework.beans.factory.BeanFactory;

import org.springframework.beans.factory.ListableBeanFactory;

import com.google.common.collect.Maps;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginEnumerator;

import eu.dnetlib.data.download.rmi.DownloadServiceException;

/**

 * The Class DownloadPluginEnumeratorImpl.

 */

public class DownloadPluginEnumeratorImpl implements DownloadPluginEnumerator {

	/**

	 * bean factory.

	 */

	private ListableBeanFactory beanFactory;

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadPluginEnumerator#getAll()

	 */

	@Override

	public Map<String, DownloadPlugin> getAll() {

		return beanFactory.getBeansOfType(DownloadPlugin.class);

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadPluginEnumerator#getByProtocols()

	 */

	@Override

	public Map<String, DownloadPlugin> getByProtocols() {

		final Map<String, DownloadPlugin> res = Maps.newHashMap();

		for (DownloadPlugin cp : getAll().values()) {

			res.put(cp.getPluginName().toLowerCase(), cp);

		}

		return res;

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see org.springframework.beans.factory.BeanFactoryAware#setBeanFactory(org.springframework.beans.factory.BeanFactory)

	 */

	@Override

	public void setBeanFactory(final BeanFactory beanFactory) throws BeansException {

		this.beanFactory = (ListableBeanFactory) beanFactory;

	}

	/**

	 * Gets the bean factory.

	 *

	 * @return the bean factory

	 */

	public ListableBeanFactory getBeanFactory() {

		return beanFactory;

	}

	/**

	 * Get given DownloadPlugin or throws exception.

	 *

	 * @param protocol

	 *            the protocol

	 * @return the download plugin

	 * @throws eu.dnetlib.data.download.rmi.DownloadServiceException

	 *             the download service exception

	 */

	public DownloadPlugin get(final String protocol) throws DownloadServiceException {

		final DownloadPlugin plugin = getByProtocols().get(protocol.toLowerCase());

		if (plugin == null) throw new DownloadServiceException("plugin not found for name: " + protocol);

		return plugin;

	}

}

package eu.dnetlib.data.download;

import java.security.KeyManagementException;

import java.security.NoSuchAlgorithmException;

import java.security.cert.CertificateException;

import java.util.ArrayList;

import java.util.List;

import javax.annotation.Resource;

import javax.net.ssl.HostnameVerifier;

import javax.net.ssl.HttpsURLConnection;

import javax.net.ssl.SSLContext;

import javax.net.ssl.SSLSession;

import javax.net.ssl.TrustManager;

import javax.net.ssl.X509TrustManager;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import org.springframework.beans.factory.annotation.Value;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadService;

import eu.dnetlib.data.download.rmi.DownloadServiceException;

import eu.dnetlib.data.download.rmi.DownloadServiceFeeder;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;

import eu.dnetlib.enabling.tools.AbstractBaseService;

import eu.dnetlib.enabling.tools.blackboard.NotificationHandler;

/**

 * The Class DownloadServiceImpl.

 */

public class DownloadServiceImpl extends AbstractBaseService implements DownloadService {

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(DownloadServiceImpl.class);

	/** The Constant END_QUEUE. */

	public static final DownloadItem END_QUEUE = new DownloadItem();

	public static final String END_QUEUE_STRING = "END_DOWNLOAD";

	@Resource

	private DownloadServiceFeeder downloadfeeder;

	/** The download plugin enumerator. */

	@Resource

	DownloadPluginEnumeratorImpl downloadPluginEnumerator;

	/** The notification handler. */

	private NotificationHandler notificationHandler;

	@Value("${services.download.http.sslcheck}")

	private boolean checkCerts;

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadService#downloadFromResultSet(javax.xml.ws.wsaddressing.W3CEndpointReference,

	 * java.lang.String, java.lang.String, java.lang.String, java.lang.String)

	 */

	@Override

	public void downloadFromResultSet(final W3CEndpointReference resultSet,

			final String plugin,

			final String objectStoreID,

			final String protocol,

			final String mimeType) throws DownloadServiceException {

		log.info(String.format("Try to download using plugin '%s' , protocol '%s' into ObjectStore '%s'", plugin, protocol, objectStoreID));

		try {

			downloadfeeder.download(resultSet.toString(), plugin, objectStoreID, protocol, mimeType, 5, null, null);

		} catch (ObjectStoreServiceException e) {

			log.error(e);

		}

		log.info(String.format("download completed using plugin '%s' , protocol '%s' into ObjectStore '%s'", plugin, protocol, objectStoreID));

	}

	/**

	 * {@inheritDoc}

	 *

	 * @see eu.dnetlib.enabling.tools.AbstractBaseService#notify(java.lang.String, java.lang.String, java.lang.String, java.lang.String)

	 */

	@Override

	public void notify(final String subscriptionId, final String topic, final String isId, final String message) {

		getNotificationHandler().notified(subscriptionId, topic, isId, message);

	}

	@Override

	public void start() {

		if (!checkCerts) {

			log.info("diabling SSL check ...");

			new Thread(new Runnable() {

				@Override

				public void run() {

					disableSslVerification();

				}

			}).start();

		}

	}

	private void disableSslVerification() {

		try {

			// Create a trust manager that does not validate certificate chains

			TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() {

				@Override

				public void checkClientTrusted(final java.security.cert.X509Certificate[] x509Certificates, final String s) throws CertificateException {

				}

				@Override

				public void checkServerTrusted(final java.security.cert.X509Certificate[] x509Certificates, final String s) throws CertificateException {

				}

				@Override

				public java.security.cert.X509Certificate[] getAcceptedIssuers() {

					return null;

				}

			} };

			// Install the all-trusting trust manager

			SSLContext sc = SSLContext.getInstance("SSL");

			sc.init(null, trustAllCerts, new java.security.SecureRandom());

			HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());

			// Create all-trusting host name verifier

			HostnameVerifier allHostsValid = new HostnameVerifier() {

				@Override

				public boolean verify(final String hostname, final SSLSession session) {

					return true;

				}

			};

			// Install the all-trusting host verifier

			HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid);

			log.info("disabled SSL check");

		} catch (NoSuchAlgorithmException e) {

			log.error(e.getMessage(), e);

		} catch (KeyManagementException e) {

			log.error(e.getMessage(), e);

		}

	}

	/**

	 * @return the downloadfeeder

	 */

	public DownloadServiceFeeder getDownloadfeeder() {

		return downloadfeeder;

	}

	/**

	 * @param downloadfeeder

	 *            the downloadfeeder to set

	 */

	public void setDownloadfeeder(final DownloadServiceFeeder downloadfeeder) {

		this.downloadfeeder = downloadfeeder;

	}

	/**

	 * @return the notificationHandler

	 */

	public NotificationHandler getNotificationHandler() {

		return notificationHandler;

	}

	/**

	 * @param notificationHandler

	 *            the notificationHandler to set

	 */

	@Required

	public void setNotificationHandler(final NotificationHandler notificationHandler) {

		this.notificationHandler = notificationHandler;

	}

	@Override

	public List<String> listPlugins() throws DownloadServiceException {

		List<String> result = new ArrayList<String>();

		result.addAll(downloadPluginEnumerator.getAll().keySet());

		return result;

	}

}

services.download.http.sslcheck=false

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws"

	xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing"

	xmlns:p="http://www.springframework.org/schema/p" xmlns:http="http://cxf.apache.org/transports/http/configuration"

	xmlns:t="http://dnetlib.eu/springbeans/t" xmlns:template="http://dnetlib.eu/springbeans/template"

	xmlns:util="http://www.springframework.org/schema/util"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd

                                    http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd

                                    http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd

                                    http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd

                            http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd

                            http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd

                            http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">

    <bean id="downloadNotificationHandler"

		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerExecutorNotificationHandler"

		p:blackboardExecutor-ref="downloadBlackboardExecutor" />

	<bean id = "DownloadServiceFeeder" class ="eu.dnetlib.data.download.rmi.DownloadServiceFeeder" />

	<bean id="downloadBlackboardExecutor"

		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerActionExecutor"

		p:blackboardHandler-ref="blackboardHandler"

		p:actionType="eu.dnetlib.data.download.rmi.DownloadServiceActions"

		p:incomplete="false">

		<property name="actionMap">

			<map>

				<entry key="DOWNLOAD">

					<bean class="eu.dnetlib.data.download.DownloadAction"/>

				</entry>				

			</map>

		</property>

	</bean>		

    <bean id="DummyPlugin" class="eu.dnetlib.data.download.plugins.DummyPlugin" />

	<bean id="downloadSerive" class="eu.dnetlib.data.download.DownloadServiceImpl"

		init-method="start" p:notificationHandler-ref="downloadNotificationHandler" />

	<bean id="downloadPluginEnumerator" class="eu.dnetlib.data.download.DownloadPluginEnumeratorImpl" />

	<jaxws:endpoint id="downloadServiceEndpoint"

		implementor="#downloadSerive" implementorClass="eu.dnetlib.data.download.rmi.DownloadService"

		address="/downloadService" />

	<template:instance name="serviceRegistrationManager"

		t:serviceRegistrationManagerClass="eu.dnetlib.enabling.tools.registration.ValidatingServiceRegistrationManagerImpl"

		t:name="downloadServiceRegistrationManager" t:service="downloadSerive"

		t:endpoint="downloadServiceEndpoint" t:jobScheduler="jobScheduler"

		t:serviceRegistrator="blackboardServiceRegistrator" />

</beans>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-download-service</artifactId>

	<version>2.1.7</version>

	<scm>

	  <developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-download-service/tags/dnet-download-service-2.1.7</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-download-service-rmi</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>		

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-service-common</artifactId>

			<version>[2.1.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-modular-objectstore-service</artifactId>

			<version>[4.0.0,5.0.0)</version>

		</dependency>		

	</dependencies>

</project>