Project

General

Profile

« Previous | Next » 

Revision 47876

[maven-release-plugin] copy for tag dnet-download-service-2.2.1

View differences:

modules/dnet-download-service/tags/dnet-download-service-2.2.1/deploy.info
1
{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-download-service/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-download-service"}
modules/dnet-download-service/tags/dnet-download-service-2.2.1/src/test/java/DownloadWorkerTest.java
1
import java.security.Security;
2

  
3
import com.google.common.base.Function;
4
import eu.dnetlib.data.download.DownloadReportMap;
5
import eu.dnetlib.data.download.rmi.DownloadItem;
6
import eu.dnetlib.data.download.worker.DownloadWorker;
7
import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord;
8
import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;
9
import eu.dnetlib.data.objectstore.rmi.Protocols;
10

  
11
import org.junit.Before;
12
import org.junit.Ignore;
13
import org.junit.Test;
14
import org.junit.runner.RunWith;
15
import org.mockito.Mock;
16
import org.mockito.junit.MockitoJUnitRunner;
17

  
18
import static org.junit.Assert.assertEquals;
19
import static org.mockito.Matchers.any;
20
import static org.mockito.Mockito.when;
21

  
22
/**
23
 * Created by alessia on 22/12/16.
24
 */
25
@RunWith(MockitoJUnitRunner.class)
26
public class DownloadWorkerTest {
27

  
28
	private DownloadWorker dw;
29
	private DownloadReportMap reportMap;
30
	@Mock
31
	private ObjectStore objectStore;
32

  
33
	@Before
34
	public void prepare() throws Exception{
35
		//Security.insertProviderAt(new BouncyCastleProvider(),1);
36
		//System.setProperty("https.protocols", "TLSv1,TLSv1.1,TLSv1.2");
37
		reportMap = new DownloadReportMap();
38

  
39
		when(objectStore.feedObjectRecord(any(ObjectStoreRecord.class))).thenReturn("OK");
40

  
41
		dw = new DownloadWorker(null, objectStore, Protocols.valueOf("HTTP"), "application/pdf", 60000, 60000, 0,
42
				new Function<String, DownloadItem>() {
43

  
44
					@Override
45
					public DownloadItem apply(final String input) {
46
						return DownloadItem.newObjectfromJSON(input);
47
					}
48
				});
49
	}
50

  
51
	@Ignore
52
	@Test
53
	public void testSSL(){
54
		String url = "https://econstor.eu/bitstream/10419/45606/1/658945068.pdf";
55
		dw.doDownload(1, reportMap, getDownloadItem("658945068.pdf", "658945068", url));
56

  
57
		System.out.println("Report map status "+reportMap.getStatus());
58
		assertEquals(1,reportMap.getTotalDownloaded());
59
	}
60

  
61
	private DownloadItem getDownloadItem(final String fileName, final String id, String url){
62
		DownloadItem item = new DownloadItem();
63
		item.setFileName(fileName);
64
		item.setIdItemMetadata(id);
65
		item.setOpenAccess("OPEN");
66
		item.setOriginalUrl(url);
67
		item.setUrl(url);
68
		return item;
69
	}
70
}
modules/dnet-download-service/tags/dnet-download-service-2.2.1/src/main/java/eu/dnetlib/data/download/DownloadReportMap.java
1
package eu.dnetlib.data.download;
2

  
3
import java.util.HashMap;
4

  
5
import com.google.gson.Gson;
6

  
7
/**
8
 * Created by sandro on 06/11/14.
9
 */
10
public class DownloadReportMap extends HashMap<String, DownloadReport> {
11

  
12
	/**
13
	 *
14
	 */
15
	private static final long serialVersionUID = -8881211500697708728L;
16

  
17
	private boolean status;
18

  
19
	private int totalDownloaded;
20

  
21
	public DownloadReportMap() {
22
		this.totalDownloaded = 0;
23
	}
24

  
25
	public void addDowload() {
26
		this.totalDownloaded++;
27
	}
28

  
29
	public boolean getStatus() {
30
		return status;
31
	}
32

  
33
	public void setStatus(final boolean status) {
34
		this.status = status;
35
	}
36

  
37
	public int getTotalDownloaded() {
38
		return totalDownloaded;
39
	}
40

  
41
	public void setTotalDownloaded(final int totalDownloaded) {
42
		this.totalDownloaded = totalDownloaded;
43
	}
44

  
45
	@Override
46
	public String toString() {
47
		Gson g = new Gson();
48
		return g.toJson(this);
49
	}
50
}
modules/dnet-download-service/tags/dnet-download-service-2.2.1/src/main/java/eu/dnetlib/data/download/plugins/DummyPlugin.java
1
package eu.dnetlib.data.download.plugins;
2

  
3
import com.google.common.base.Function;
4
import com.google.common.collect.Iterables;
5
import com.google.gson.Gson;
6
import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;
7
import eu.dnetlib.data.download.rmi.DownloadItem;
8
import eu.dnetlib.data.download.rmi.DownloadPlugin;
9

  
10
import java.util.ArrayList;
11
import java.util.List;
12

  
13
public class DummyPlugin extends AbstractDownloadPlugin implements DownloadPlugin {
14

  
15

  
16
    @Override
17
    public void setBasePath(String basePath) {
18

  
19
    }
20

  
21
    @Override
22
    public DownloadItem retrieveUrl(final DownloadItem item) {
23
        if (checkOpenAccess(item) == null) {
24
            return null;
25
        }
26
        if (filterByRegexp(item) == null) return null;
27
        String url = item.getUrl();
28

  
29
        if ((url == null) || (url.trim().length() == 0)) return null;
30
        @SuppressWarnings("unchecked")
31
        List<String> urls = new Gson().fromJson(url, ArrayList.class);
32
        if ((urls == null) || (urls.size() == 0)) {
33
            item.setOriginalUrl(null);
34
            item.setUrl(null);
35
            return item;
36
        }
37
        item.setOriginalUrl(urls.get(0));
38
        item.setUrl(urls.get(0));
39
        return item;
40
    }
41

  
42
    @Override
43
    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> items) {
44
        return Iterables.transform(items, new Function<DownloadItem, DownloadItem>() {
45

  
46
            @Override
47
            public DownloadItem apply(final DownloadItem input) {
48
                return retrieveUrl(input);
49
            }
50
        });
51
    }
52

  
53
    @Override
54
    public String getPluginName() {
55
        return "DummyPlugin";
56
    }
57

  
58

  
59
    @Override
60
    public String extractURL(String baseURL) {
61
        return null;
62
    }
63
}
modules/dnet-download-service/tags/dnet-download-service-2.2.1/src/main/java/eu/dnetlib/data/download/DownloadReport.java
1
package eu.dnetlib.data.download;
2

  
3
import eu.dnetlib.data.download.rmi.DownloadItem;
4

  
5
/**
6
 * Created by sandro on 06/11/14.
7
 */
8
public class DownloadReport {
9

  
10
    private int numberOfOccurrences;
11
    private String stackTrace;
12

  
13
	private DownloadItem downloadItem;
14

  
15

  
16
    public DownloadReport() {
17
        this.numberOfOccurrences =1;
18

  
19
    }
20

  
21
	public DownloadReport(int numberOfOccurrences, String stackTrace, DownloadItem downloadItem) {
22
		this.numberOfOccurrences = numberOfOccurrences;
23
        this.stackTrace = stackTrace;
24
		this.downloadItem = downloadItem;
25
	}
26

  
27
    public void incrementError(){
28
        this.numberOfOccurrences ++;
29
    }
30

  
31
    public void incrementError(int numberOfOccurrences){
32
        this.numberOfOccurrences += numberOfOccurrences;
33
    }
34

  
35
    /**
36
     * Getter for property 'numberOfOccurrences'.
37
     *
38
     * @return Value for property 'numberOfOccurrences'.
39
     */
40
    public int getNumberOfOccurrences() {
41

  
42
        return numberOfOccurrences;
43
    }
44

  
45
    /**
46
     * Setter for property 'numberOfOccurrences'.
47
     *
48
     * @param numberOfOccurrences Value to set for property 'numberOfOccurrences'.
49
     */
50
    public void setNumberOfOccurrences(int numberOfOccurrences) {
51
        this.numberOfOccurrences = numberOfOccurrences;
52
    }
53

  
54
    /**
55
     * Getter for property 'stackTrace'.
56
     *
57
     * @return Value for property 'stackTrace'.
58
     */
59
    public String getStackTrace() {
60
        return stackTrace;
61
    }
62

  
63
    /**
64
     * Setter for property 'stackTrace'.
65
     *
66
     * @param stackTrace Value to set for property 'stackTrace'.
67
     */
68
    public void setStackTrace(String stackTrace) {
69
        this.stackTrace = stackTrace;
70
    }
71

  
72
	public DownloadItem getDownloadItem() {
73
		return downloadItem;
74
	}
75

  
76
	public void setDownloadItem(final DownloadItem downloadItem) {
77
		this.downloadItem = downloadItem;
78
	}
79
}
modules/dnet-download-service/tags/dnet-download-service-2.2.1/src/main/java/eu/dnetlib/data/download/DownloadAction.java
1
package eu.dnetlib.data.download;
2

  
3
import java.util.List;
4
import javax.annotation.Resource;
5

  
6
import com.google.gson.Gson;
7
import com.google.gson.reflect.TypeToken;
8
import eu.dnetlib.data.download.rmi.DownloadServiceActions;
9
import eu.dnetlib.data.download.rmi.DownloadServiceException;
10
import eu.dnetlib.data.download.rmi.DownloadServiceFeeder;
11
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
12
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerAction;
13
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
14
import org.apache.commons.codec.binary.Base64;
15
import org.apache.commons.lang.StringUtils;
16
import org.apache.commons.logging.Log;
17
import org.apache.commons.logging.LogFactory;
18

  
19
public class DownloadAction implements BlackboardServerAction<DownloadServiceActions> {
20

  
21
	/** The Constant log. */
22
	private static final Log log = LogFactory.getLog(DownloadAction.class);
23
	private static String DEFAULT_NUMBER_OF_THREAD = "5";
24
	private static int DEFAULT_TIMEOUT_MS = 5000;
25
	@Resource
26
	private DownloadServiceFeeder downloadServiceFeeder;
27

  
28
	@Override
29
	public void execute(final BlackboardServerHandler handler, final BlackboardJob job) throws Exception {
30
		int connectTimeoutMs = DEFAULT_TIMEOUT_MS;
31
		int readTimeoutMs = DEFAULT_TIMEOUT_MS;
32
		int sleepTimeMs = 0;
33
		try {
34
			String epr = job.getParameters().get("epr");
35
			String plugin = job.getParameters().get("plugin");
36
			String objectStoreID = job.getParameters().get("objectStoreID");
37
			String protocol = job.getParameters().get("protocol");
38
			String mimeType = job.getParameters().get("mimeType");
39
			String numberOfThreads = job.getParameters().get("numberOfThreads");
40
			String basePath = job.getParameters().get("basePath");
41
			String regularExpression = job.getParameters().get("regularExpressions");
42
			String connectTimeoutMsStr = job.getParameters().get("connectTimeoutMs");
43
			String readTimeoutMsStr = job.getParameters().get("readTimeoutMs");
44
			String sleepTimeMsStr = job.getParameters().get("sleepTimeMs");
45
			log.debug("regular Expression: " + regularExpression);
46

  
47
			List<String> expressions = null;
48

  
49
			if (!StringUtils.isBlank(regularExpression)) {
50
				expressions = parseRegexList(regularExpression);
51
			}
52

  
53
			if (StringUtils.isBlank(numberOfThreads)) {
54
				log.warn("Cannot find numberOfThread, using default value " + DEFAULT_NUMBER_OF_THREAD);
55
				numberOfThreads = DEFAULT_NUMBER_OF_THREAD;
56
			}
57

  
58
			if (StringUtils.isNotBlank(connectTimeoutMsStr)) connectTimeoutMs = Integer.parseInt(connectTimeoutMsStr);
59
			if (StringUtils.isNotBlank(readTimeoutMsStr)) readTimeoutMs = Integer.parseInt(readTimeoutMsStr);
60
			if (StringUtils.isNotBlank(sleepTimeMsStr)) sleepTimeMs = Integer.parseInt(sleepTimeMsStr);
61

  
62
			log.info(String.format("downloading using plugin %s , protocol : %s into ObjectStore id %s  ", plugin, protocol, objectStoreID));
63
			handler.ongoing(job);
64
			DownloadReportMap response = downloadServiceFeeder.download(epr, plugin, objectStoreID, protocol, mimeType, Integer.parseInt(numberOfThreads),
65
					basePath, expressions, connectTimeoutMs, readTimeoutMs, sleepTimeMs);
66
			if (response.getStatus()) {
67
				log.info(String.format("Completed Download, plugin: %s, protocol: %s, objectStoreID: %s", plugin, protocol, objectStoreID));
68
				job.getParameters().put("total", "" + response.getTotalDownloaded());
69
				job.getParameters().put("report", Base64.encodeBase64String(response.toString().getBytes()));
70
				handler.done(job);
71
			} else {
72
				log.error("download response is false" + response);
73

  
74
				handler.failed(job, new DownloadServiceException(
75
						"oops! something bad happen to our download hamsters, downloaded so far: " + response.getTotalDownloaded()));
76
			}
77
		} catch (Exception e) {
78
			log.error("An error occur while starting download", e);
79
			handler.failed(job, new DownloadServiceException(e));
80
		}
81
	}
82

  
83
	private List<String> parseRegexList(final String regularExpression) {
84
		log.info("parsing regex list: " + regularExpression);
85
		return new Gson().fromJson(regularExpression, new TypeToken<List<String>>() {}.getType());
86
	}
87

  
88
}
modules/dnet-download-service/tags/dnet-download-service-2.2.1/src/main/java/eu/dnetlib/data/download/rmi/DownloadServiceFeeder.java
1
package eu.dnetlib.data.download.rmi;
2

  
3
import java.util.List;
4
import java.util.concurrent.BlockingQueue;
5
import java.util.concurrent.ExecutorService;
6
import java.util.concurrent.Executors;
7
import java.util.concurrent.Future;
8
import javax.annotation.Resource;
9

  
10
import com.google.common.base.Function;
11
import com.google.common.base.Joiner;
12
import com.google.common.collect.Lists;
13
import com.google.common.collect.Queues;
14
import eu.dnetlib.data.download.DownloadPluginEnumeratorImpl;
15
import eu.dnetlib.data.download.DownloadReport;
16
import eu.dnetlib.data.download.DownloadReportMap;
17
import eu.dnetlib.data.download.DownloadServiceImpl;
18
import eu.dnetlib.data.download.worker.DownloadWorker;
19
import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;
20
import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
21
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
22
import eu.dnetlib.data.objectstore.rmi.Protocols;
23
import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
24
import org.apache.commons.logging.Log;
25
import org.apache.commons.logging.LogFactory;
26
import org.springframework.beans.factory.annotation.Autowired;
27

  
28
/**
29
 * The Class DownloadServiceFeeder.
30
 */
31
public class DownloadServiceFeeder {
32

  
33
	/** The Constant log. */
34
	private static final Log log = LogFactory.getLog(DownloadServiceFeeder.class);
35
	private static final int MAX_NUM_FOUND = 10;
36

  
37
	/** The download plugin enumerator. */
38
	@Resource
39
	DownloadPluginEnumeratorImpl downloadPluginEnumerator;
40

  
41
	/** The result set client factory. */
42
	@Resource
43
	private ResultSetClientFactory resultSetClientFactory;
44

  
45
	/** The object store dao. */
46
	@Autowired
47
	private ObjectStoreDao objectStoreDao;
48

  
49
	public static void reportException(final DownloadReportMap report, final DownloadItem di, final Throwable e) {
50
		final String className = e.getClass().getName();
51
		if (!report.containsKey(className)) {
52
			final DownloadReport dr = new DownloadReport();
53
			dr.setStackTrace(Joiner.on("\tat ").join(e.getStackTrace()));
54
			if (di != null) {
55
				dr.setDownloadItem(di);
56
			}
57
			report.put(className, dr);
58
		} else {
59
			report.get(className).incrementError();
60
		}
61
	}
62

  
63
	/**
64
	 * Download and feed file into the objectStore .
65
	 *
66
	 * @param epr
67
	 *            the end-point reference of the result-set of Serialized DownloadItem
68
	 * @param plugin
69
	 *            the plugin used to retrieve the correct URL
70
	 * @param objectStoreID
71
	 *            the object store id to store the data
72
	 * @param protocol
73
	 *            the protocol used to download the file
74
	 * @param mimeType
75
	 *            the mime type of the Files
76
	 * @param numberOfThreads
77
	 *            the number of threads to use for download at the same time
78
	 * @throws DownloadServiceException
79
	 *             the download service exception
80
	 * @throws ObjectStoreServiceException
81
	 */
82
	public DownloadReportMap download(final String epr,
83
			final String plugin,
84
			final String objectStoreID,
85
			final String protocol,
86
			final String mimeType,
87
			final int numberOfThreads,
88
			final String basePath,
89
			final List<String> regularExpression,
90
			final int connectTimeoutMs,
91
			final int readTimeoutMs, final int sleepTimeMs) throws DownloadServiceException, ObjectStoreServiceException {
92
		final DownloadPlugin downloadPlugin = downloadPluginEnumerator.get(plugin);
93
		if ((basePath != null) && (basePath.isEmpty() == false)) {
94
			downloadPlugin.setBasePath(basePath);
95
		}
96

  
97
		final Iterable<String> urlInfo = resultSetClientFactory.getClient(epr);
98
		final BlockingQueue<String> itemsQueue = Queues.newArrayBlockingQueue(1024);
99
		final ExecutorService executor = Executors.newFixedThreadPool(numberOfThreads);
100
		final ObjectStore objStore = objectStoreDao.getObjectStore(objectStoreID);
101

  
102
		final List<Future<DownloadReportMap>> responses = Lists.newArrayList();
103
		final DownloadReportMap pluginReport = new DownloadReportMap();
104
		pluginReport.setStatus(true);
105

  
106
		if (regularExpression != null) {
107
			downloadPlugin.setRegularExpression(regularExpression);
108
		}
109

  
110
		for (int i = 0; i < numberOfThreads; i++) {
111
			responses.add(executor
112
					.submit(new DownloadWorker(itemsQueue, objStore, Protocols.valueOf(protocol), mimeType, connectTimeoutMs, readTimeoutMs, sleepTimeMs,
113
					new Function<String, DownloadItem>() {
114

  
115
				@Override
116
				public DownloadItem apply(final String input) {
117
					if (input == null) {
118
						log.error("Input is null");
119
						return null;
120
					}
121
					if (input.equals(DownloadServiceImpl.END_QUEUE_STRING)) return DownloadServiceImpl.END_QUEUE;
122

  
123
					DownloadItem di = null;
124
					try {
125
						di = DownloadItem.newObjectfromJSON(input);
126

  
127
						if (downloadPlugin.retrieveUrl(di) == null) {
128
							di.setUrl(null);
129
							di.setOriginalUrl(null);
130
						}
131
						return di;
132
					} catch (Throwable e) {
133
						reportException(pluginReport, di, e);
134
						log.error("Exception on transform item :" + input, e);
135
						return null;
136
					}
137
				}
138
			})));
139
		}
140

  
141
		int i = 0;
142
		int null_found = 0;
143
		if (urlInfo != null) {
144
			for (final String downloadItem : urlInfo) {
145
				if (downloadItem != null) {
146
					null_found = 0;
147
					if ((i++ % 1000) == 0) {
148
						log.debug("Read " + i);
149
					}
150
					try {
151
						itemsQueue.put(downloadItem);
152
					} catch (final Exception e) {
153
						log.error("An error occurred while populating the download items queue: " + Joiner.on("\tat ").join(e.getStackTrace()));
154
					}
155
				} else {
156
					if (null_found++ > MAX_NUM_FOUND) {
157
						break;
158
					}
159
				}
160

  
161
			}
162
		}
163

  
164
		try {
165
			itemsQueue.put(DownloadServiceImpl.END_QUEUE_STRING);
166
		} catch (final InterruptedException e) {
167
			log.error("An error occurred adding the loop terminator: " + Joiner.on("\tat ").join(e.getStackTrace()));
168
		}
169

  
170
		final DownloadReportMap resultMap = getDownloadReportMap(responses, pluginReport);
171
		executor.shutdown();
172
		return resultMap;
173
	}
174

  
175
	private DownloadReportMap getDownloadReportMap(final List<Future<DownloadReportMap>> responses, final DownloadReportMap pluginReport) {
176
		final DownloadReportMap resultMap = new DownloadReportMap();
177
		resultMap.setStatus(true);
178

  
179
		for (final Future<DownloadReportMap> currentResponse : responses) {
180
			try {
181
				final DownloadReportMap currentMap = currentResponse.get();
182

  
183
				mergeReport(resultMap, currentMap);
184

  
185
				log.info("Status " + currentMap.getStatus());
186
				resultMap.setStatus(resultMap.getStatus() && currentMap.getStatus());
187
				resultMap.setTotalDownloaded(currentMap.getTotalDownloaded() + resultMap.getTotalDownloaded());
188

  
189
			} catch (final Exception e) {
190
				log.error(e);
191
				resultMap.setStatus(false);
192
			}
193
		}
194
		mergeReport(resultMap, pluginReport);
195
		return resultMap;
196
	}
197

  
198
	private void mergeReport(final DownloadReportMap resultMap, final DownloadReportMap currentMap) {
199
		for (final String key : currentMap.keySet()) {
200
			if (!resultMap.containsKey(key)) {
201
				resultMap.put(key, currentMap.get(key));
202
			} else {
203
				final DownloadReport currentReport = currentMap.get(key);
204
				resultMap.get(key).incrementError(currentReport.getNumberOfOccurrences());
205
			}
206
			resultMap.setTotalDownloaded(resultMap.getTotalDownloaded() + currentMap.getTotalDownloaded());
207
			resultMap.setStatus(resultMap.getStatus() & currentMap.getStatus());
208
		}
209
	}
210
}
modules/dnet-download-service/tags/dnet-download-service-2.2.1/src/main/java/eu/dnetlib/data/download/rmi/DownloadServiceActions.java
1
package eu.dnetlib.data.download.rmi;
2

  
3
public enum DownloadServiceActions {
4
	DOWNLOAD,
5

  
6
}
modules/dnet-download-service/tags/dnet-download-service-2.2.1/src/main/java/eu/dnetlib/data/download/worker/DownloadWorker.java
1
package eu.dnetlib.data.download.worker;
2

  
3
import java.io.IOException;
4
import java.net.HttpURLConnection;
5
import java.net.URL;
6
import java.net.URLConnection;
7
import java.util.concurrent.BlockingQueue;
8
import java.util.concurrent.Callable;
9
import java.util.concurrent.TimeUnit;
10

  
11
import com.google.common.base.Function;
12
import com.google.common.base.Joiner;
13
import eu.dnetlib.data.download.DownloadReportMap;
14
import eu.dnetlib.data.download.DownloadServiceImpl;
15
import eu.dnetlib.data.download.rmi.DownloadItem;
16
import eu.dnetlib.data.download.rmi.DownloadServiceFeeder;
17
import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord;
18
import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;
19
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
20
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
21
import eu.dnetlib.data.objectstore.rmi.Protocols;
22
import org.apache.commons.lang.StringUtils;
23
import org.apache.commons.logging.Log;
24
import org.apache.commons.logging.LogFactory;
25

  
26
/**
27
 * The Class DownloadWorker is a worker responsible to download the data into the object store.
28
 */
29
public class DownloadWorker implements Callable<DownloadReportMap> {
30

  
31
	/**
32
	 * The Constant log.
33
	 */
34
	private static final Log log = LogFactory.getLog(DownloadWorker.class);
35
	private static final int MAX_NULLS = 5 ;
36

  
37
	/**
38
	 * The queue.
39
	 */
40
	private BlockingQueue<String> queue = null;
41

  
42
	/**
43
	 * The object store.
44
	 */
45
	private ObjectStore objectStore = null;
46

  
47
	/**
48
	 * The protocol.
49
	 */
50
	private Protocols protocol;
51

  
52
	/**
53
	 * The mime type.
54
	 */
55
	private String mimeType;
56

  
57
	private Function<String, DownloadItem> converter;
58

  
59
	private int connectTimeoutMs;
60
	private int readTimeoutMs;
61

  
62
	/**
63
	 * ms to wait between two subsequent download request
64
	 **/
65
	private int sleepTimeMs;
66

  
67
	/**
68
	 * Instantiates a new download worker.
69
	 *
70
	 * @param queue
71
	 *            the queue
72
	 * @param objectStore
73
	 *            the object store
74
	 * @param protocol
75
	 *            the protocol
76
	 * @param mimeType
77
	 *            the mime type
78
	 */
79
	public DownloadWorker(final BlockingQueue<String> queue,
80
			final ObjectStore objectStore,
81
			final Protocols protocol,
82
			final String mimeType,
83
			final int connectTimeoutMs,
84
			final int readTimeoutMs, final int sleepTimeMs,
85
			final Function<String, DownloadItem> converter) {
86
		this.setConverter(converter);
87
		this.setQueue(queue);
88
		this.setObjectStore(objectStore);
89
		this.setMimeType(mimeType);
90
		this.setProtocol(protocol);
91
		this.setConnectTimeoutMs(connectTimeoutMs);
92
		this.setReadTimeoutMs(readTimeoutMs);
93
		this.setSleepTimeMs(sleepTimeMs);
94
	}
95

  
96
	/*
97
	 * (non-Javadoc)
98
	 *
99
	 * @see java.lang.Runnable#run()
100
	 */
101
	@Override
102
	public DownloadReportMap call() throws Exception {
103
		final DownloadReportMap report = new DownloadReportMap();
104
		final long threadId = Thread.currentThread().getId();
105
		int nullCounter = 0;
106
		try {
107

  
108
			String takedObject = queue.poll(5, TimeUnit.SECONDS);
109

  
110
			while (!DownloadServiceImpl.END_QUEUE_STRING.equals(takedObject) && nullCounter < MAX_NULLS) {
111

  
112
				if (takedObject == null) {
113
					nullCounter++;
114
				}
115
				if (log.isDebugEnabled()) {
116
					log.debug(takedObject);
117
				}
118
				final DownloadItem di = getConverter().apply(takedObject);
119

  
120
				log.debug(threadId + ": Reading " + takedObject);
121

  
122
				if (di == null) {
123
					log.info("the current download item is Null, skipping");
124
					DownloadServiceFeeder.reportException(report, null, new IllegalArgumentException("found null DownloadItem"));
125
				} else if (StringUtils.isNotBlank(di.getUrl()) && !checkIfExists(di)) {
126
					doDownload(threadId, report, di);
127
				}
128
				takedObject = queue.poll(5, TimeUnit.SECONDS);
129
				log.debug(String.format("%s: next object from queue %s, remaining items: %s", threadId, takedObject, queue.size()));
130
			}
131
		} catch (final Exception e) {
132
			log.debug("An error occured : " + Joiner.on("\tat ").join(e.getStackTrace()));
133
			DownloadServiceFeeder.reportException(report, null, e);
134
			report.setStatus(false);
135
			return report;
136
		} finally {
137
			log.info(String.format("%s: finalising queue, remaining items: %s, nulls: %s", threadId, queue.size(), nullCounter));
138
			final boolean res = queue.offer(DownloadServiceImpl.END_QUEUE_STRING, 5, TimeUnit.SECONDS);
139
			log.info("put terminator in queue: " + res);
140
		}
141

  
142
		log.info("CLOSED THREAD " + threadId);
143
		report.setStatus(true);
144
		return report;
145
	}
146

  
147
	public void doDownload(final long threadId, final DownloadReportMap report, final DownloadItem di) {
148
		try {
149
			if (getSleepTimeMs() > 0) {
150
				log.debug(threadId + ": I will sleep for " + getSleepTimeMs() + " ms, as requested...");
151
				Thread.sleep(getSleepTimeMs());
152
			}
153

  
154
			final URL toDownload = followURL(threadId, new URL(di.getUrl()), report, di);
155
			final ObjectStoreRecord record = new ObjectStoreRecord();
156
			final ObjectStoreFile metadata = new ObjectStoreFile();
157
			metadata.setObjectID(di.getFileName());
158
			metadata.setMetadataRelatedID(di.getIdItemMetadata());
159
			metadata.setAccessProtocol(this.protocol);
160
			metadata.setMimeType(this.mimeType);
161
			metadata.setDownloadedURL(di.getOriginalUrl());
162
			record.setFileMetadata(metadata);
163

  
164
			log.debug(threadId + ": opening connection " + toDownload);
165
			final URLConnection connection = toDownload.openConnection();
166
			connection.setConnectTimeout(getConnectTimeoutMs());
167
			connection.setReadTimeout(getReadTimeoutMs());
168

  
169
			log.debug(threadId + ": getting input stream from " + toDownload);
170
			record.setInputStream(connection.getInputStream());
171
			log.debug(threadId + ": feeding object from  " + toDownload + " into objectstore ...");
172
			objectStore.feedObjectRecord(record);
173
			report.addDowload();
174
			log.debug(threadId + ": saved object " + metadata.toJSON());
175
		} catch (final Throwable e) {
176
			log.debug(threadId + ": error downloading Item: " + di.toJSON(), e);
177
			DownloadServiceFeeder.reportException(report, di, e);
178
		}
179
	}
180

  
181
	private URL followURL(final long threadId, final URL inputURL, final DownloadReportMap report, final DownloadItem di) throws IOException {
182

  
183
		final String ptrcl = inputURL.getProtocol();
184
		if (ptrcl.startsWith("file")) {
185
			log.debug("the protocol is File, returning " + inputURL);
186
			return inputURL;
187
		}
188
		HttpURLConnection conn = (HttpURLConnection) inputURL.openConnection();
189
		conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manually.
190
		conn.setReadTimeout(getReadTimeoutMs());
191
		conn.setConnectTimeout(getConnectTimeoutMs());
192
		String location = inputURL.toString();
193
		log.debug(threadId + " : followURL connecting  " + inputURL);
194
		conn.connect();
195
		log.debug(threadId + " : followURL connected  " + inputURL);
196
		int responseCode = conn.getResponseCode();
197
		log.debug(threadId + " : followURL " + inputURL + ", response code: " + responseCode);
198
		if ((responseCode >= 300) && (responseCode < 400)) {
199
			location = conn.getHeaderFields().get("Location").get(0);
200
			conn.disconnect();
201
			log.debug(threadId + " : followURL disconnected  " + inputURL);
202
		}
203

  
204
		if (!location.equals(inputURL.toString())) return new URL(location);
205
		return inputURL;
206
	}
207

  
208
	private boolean checkIfExists(final DownloadItem di) {
209
		try {
210
			return objectStore.deliverObject(di.getFileName()) != null;
211
		} catch (final ObjectStoreServiceException e) {
212
			log.debug(e.getMessage());
213
			return false;
214
		}
215
	}
216

  
217
	/**
218
	 * Sets the object store.
219
	 *
220
	 * @param objectStore
221
	 *            the objectStore to set
222
	 */
223
	public void setObjectStore(final ObjectStore objectStore) {
224
		this.objectStore = objectStore;
225
	}
226

  
227
	/**
228
	 * Sets the queue.
229
	 *
230
	 * @param queue
231
	 *            the queue to set
232
	 */
233
	public void setQueue(final BlockingQueue<String> queue) {
234
		this.queue = queue;
235
	}
236

  
237
	/**
238
	 * Sets the protocol.
239
	 *
240
	 * @param protocol
241
	 *            the protocol to set
242
	 */
243
	public void setProtocol(final Protocols protocol) {
244
		this.protocol = protocol;
245
	}
246

  
247
	/**
248
	 * Sets the mime type.
249
	 *
250
	 * @param mimeType
251
	 *            the mimeType to set
252
	 */
253
	public void setMimeType(final String mimeType) {
254
		this.mimeType = mimeType;
255
	}
256

  
257
	public Function<String, DownloadItem> getConverter() {
258
		return converter;
259
	}
260

  
261
	public void setConverter(final Function<String, DownloadItem> converter) {
262
		this.converter = converter;
263
	}
264

  
265
	public int getReadTimeoutMs() {
266
		return readTimeoutMs;
267
	}
268

  
269
	public void setReadTimeoutMs(final int readTimeoutMs) {
270
		this.readTimeoutMs = readTimeoutMs;
271
	}
272

  
273
	public int getConnectTimeoutMs() {
274
		return connectTimeoutMs;
275
	}
276

  
277
	public void setConnectTimeoutMs(final int connectTimeoutMs) {
278
		this.connectTimeoutMs = connectTimeoutMs;
279
	}
280

  
281
	public int getSleepTimeMs() {
282
		return sleepTimeMs;
283
	}
284

  
285
	public void setSleepTimeMs(final int sleepTimeMs) {
286
		this.sleepTimeMs = sleepTimeMs;
287
	}
288
}
modules/dnet-download-service/tags/dnet-download-service-2.2.1/src/main/java/eu/dnetlib/data/download/DownloadPluginEnumeratorImpl.java
1
package eu.dnetlib.data.download;
2

  
3
import java.util.Map;
4

  
5
import org.springframework.beans.BeansException;
6
import org.springframework.beans.factory.BeanFactory;
7
import org.springframework.beans.factory.ListableBeanFactory;
8

  
9
import com.google.common.collect.Maps;
10

  
11
import eu.dnetlib.data.download.rmi.DownloadPlugin;
12
import eu.dnetlib.data.download.rmi.DownloadPluginEnumerator;
13
import eu.dnetlib.data.download.rmi.DownloadServiceException;
14

  
15
/**
16
 * The Class DownloadPluginEnumeratorImpl.
17
 */
18
public class DownloadPluginEnumeratorImpl implements DownloadPluginEnumerator {
19

  
20
	/**
21
	 * bean factory.
22
	 */
23
	private ListableBeanFactory beanFactory;
24

  
25
	/*
26
	 * (non-Javadoc)
27
	 *
28
	 * @see eu.dnetlib.data.download.rmi.DownloadPluginEnumerator#getAll()
29
	 */
30
	@Override
31
	public Map<String, DownloadPlugin> getAll() {
32
		return beanFactory.getBeansOfType(DownloadPlugin.class);
33
	}
34

  
35
	/*
36
	 * (non-Javadoc)
37
	 *
38
	 * @see eu.dnetlib.data.download.rmi.DownloadPluginEnumerator#getByProtocols()
39
	 */
40
	@Override
41
	public Map<String, DownloadPlugin> getByProtocols() {
42
		final Map<String, DownloadPlugin> res = Maps.newHashMap();
43
		for (DownloadPlugin cp : getAll().values()) {
44
			res.put(cp.getPluginName().toLowerCase(), cp);
45
		}
46
		return res;
47
	}
48

  
49
	/*
50
	 * (non-Javadoc)
51
	 *
52
	 * @see org.springframework.beans.factory.BeanFactoryAware#setBeanFactory(org.springframework.beans.factory.BeanFactory)
53
	 */
54
	@Override
55
	public void setBeanFactory(final BeanFactory beanFactory) throws BeansException {
56
		this.beanFactory = (ListableBeanFactory) beanFactory;
57
	}
58

  
59
	/**
60
	 * Gets the bean factory.
61
	 *
62
	 * @return the bean factory
63
	 */
64
	public ListableBeanFactory getBeanFactory() {
65
		return beanFactory;
66
	}
67

  
68
	/**
69
	 * Get given DownloadPlugin or throws exception.
70
	 *
71
	 * @param protocol
72
	 *            the protocol
73
	 * @return the download plugin
74
	 * @throws eu.dnetlib.data.download.rmi.DownloadServiceException
75
	 *             the download service exception
76
	 */
77
	public DownloadPlugin get(final String protocol) throws DownloadServiceException {
78
		final DownloadPlugin plugin = getByProtocols().get(protocol.toLowerCase());
79
		if (plugin == null) throw new DownloadServiceException("plugin not found for name: " + protocol);
80
		return plugin;
81
	}
82

  
83
}
modules/dnet-download-service/tags/dnet-download-service-2.2.1/src/main/java/eu/dnetlib/data/download/DownloadServiceImpl.java
1
package eu.dnetlib.data.download;
2

  
3
import java.security.KeyManagementException;
4
import java.security.NoSuchAlgorithmException;
5
import java.security.Security;
6
import java.security.cert.CertificateException;
7
import java.util.ArrayList;
8
import java.util.List;
9
import javax.annotation.Resource;
10
import javax.net.ssl.*;
11
import javax.xml.ws.wsaddressing.W3CEndpointReference;
12

  
13
import eu.dnetlib.data.download.rmi.DownloadItem;
14
import eu.dnetlib.data.download.rmi.DownloadService;
15
import eu.dnetlib.data.download.rmi.DownloadServiceException;
16
import eu.dnetlib.data.download.rmi.DownloadServiceFeeder;
17
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
18
import eu.dnetlib.enabling.tools.AbstractBaseService;
19
import eu.dnetlib.enabling.tools.blackboard.NotificationHandler;
20
import org.apache.commons.logging.Log;
21
import org.apache.commons.logging.LogFactory;
22
import org.springframework.beans.factory.annotation.Required;
23
import org.springframework.beans.factory.annotation.Value;
24

  
25
/**
26
 * The Class DownloadServiceImpl.
27
 */
28
public class DownloadServiceImpl extends AbstractBaseService implements DownloadService {
29

  
30
	/**
31
	 * The Constant END_QUEUE.
32
	 */
33
	public static final DownloadItem END_QUEUE = new DownloadItem();
34

  
35
	public static final String END_QUEUE_STRING = "END_DOWNLOAD";
36

  
37
	/**
38
	 * The Constant log.
39
	 */
40
	private static final Log log = LogFactory.getLog(DownloadServiceImpl.class);
41
	private static int DEFAULT_CONNECT_TIMEOUT_MS = 5000;
42
	private static int DEFAULT_READ_TIMEOUT_MS = 5000;
43
	/**
44
	 * The download plugin enumerator.
45
	 */
46
	@Resource
47
	private DownloadPluginEnumeratorImpl downloadPluginEnumerator;
48
	@Resource
49
	private DownloadServiceFeeder downloadfeeder;
50
	/**
51
	 * The notification handler.
52
	 */
53
	private NotificationHandler notificationHandler;
54
	@Value("${services.download.http.sslcheck}")
55
	private boolean checkCerts;
56
	@Value("${services.download.https.protocols}")
57
	private String httpsProtocols;
58

  
59
	/*
60
	 * (non-Javadoc)
61
	 *
62
	 * @see eu.dnetlib.data.download.rmi.DownloadService#downloadFromResultSet(javax.xml.ws.wsaddressing.W3CEndpointReference,
63
	 * java.lang.String, java.lang.String, java.lang.String, java.lang.String)
64
	 */
65
	@Override
66
	public void downloadFromResultSet(final W3CEndpointReference resultSet,
67
			final String plugin,
68
			final String objectStoreID,
69
			final String protocol,
70
			final String mimeType) throws DownloadServiceException {
71

  
72
		log.info(String.format("download using plugin '%s' , protocol '%s' into ObjectStore '%s'", plugin, protocol, objectStoreID));
73
		try {
74
			downloadfeeder.download(resultSet.toString(), plugin, objectStoreID, protocol, mimeType, 5, null, null, DEFAULT_CONNECT_TIMEOUT_MS,
75
					DEFAULT_READ_TIMEOUT_MS, 0);
76
		} catch (ObjectStoreServiceException e) {
77
			log.error(e);
78
		}
79
		log.info(String.format("download completed using plugin '%s' , protocol '%s' into ObjectStore '%s'", plugin, protocol, objectStoreID));
80
	}
81

  
82
	/**
83
	 * {@inheritDoc}
84
	 *
85
	 * @see eu.dnetlib.enabling.tools.AbstractBaseService#notify(java.lang.String, java.lang.String, java.lang.String, java.lang.String)
86
	 */
87
	@Override
88
	public void notify(final String subscriptionId, final String topic, final String isId, final String message) {
89
		getNotificationHandler().notified(subscriptionId, topic, isId, message);
90
	}
91

  
92
	@Override
93
	public void start() {
94
		if (!checkCerts) {
95
			log.info("disabling SSL check ...");
96
			new Thread(new Runnable() {
97

  
98
				@Override
99
				public void run() {
100
					disableSslVerification();
101
				}
102
			}).start();
103
		}
104
		//TODO: we should remove references to BouncyCastle. We are not expecting issues like #2520 with Java8
105
		//Security.insertProviderAt(new BouncyCastleProvider(),1);
106
		//System.setProperty("https.protocols", httpsProtocols);
107
	}
108

  
109
	private void disableSslVerification() {
110
		try {
111
			// Create a trust manager that does not validate certificate chains
112
			TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() {
113

  
114
				@Override
115
				public void checkClientTrusted(final java.security.cert.X509Certificate[] x509Certificates, final String s) throws CertificateException {
116

  
117
				}
118

  
119
				@Override
120
				public void checkServerTrusted(final java.security.cert.X509Certificate[] x509Certificates, final String s) throws CertificateException {
121

  
122
				}
123

  
124
				@Override
125
				public java.security.cert.X509Certificate[] getAcceptedIssuers() {
126
					return null;
127
				}
128
			} };
129

  
130
			// Install the all-trusting trust manager
131
			SSLContext sc = SSLContext.getInstance("SSL");
132
			sc.init(null, trustAllCerts, new java.security.SecureRandom());
133
			HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
134

  
135
			// Create all-trusting host name verifier
136
			HostnameVerifier allHostsValid = new HostnameVerifier() {
137

  
138
				@Override
139
				public boolean verify(final String hostname, final SSLSession session) {
140
					return true;
141
				}
142
			};
143

  
144
			// Install the all-trusting host verifier
145
			HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid);
146
			log.info("disabled SSL check");
147
		} catch (NoSuchAlgorithmException e) {
148
			log.error(e.getMessage(), e);
149
		} catch (KeyManagementException e) {
150
			log.error(e.getMessage(), e);
151
		}
152
	}
153

  
154
	/**
155
	 * @return the downloadfeeder
156
	 */
157
	public DownloadServiceFeeder getDownloadfeeder() {
158
		return downloadfeeder;
159
	}
160

  
161
	/**
162
	 * @param downloadfeeder the downloadfeeder to set
163
	 */
164
	public void setDownloadfeeder(final DownloadServiceFeeder downloadfeeder) {
165
		this.downloadfeeder = downloadfeeder;
166
	}
167

  
168
	/**
169
	 * @return the notificationHandler
170
	 */
171
	public NotificationHandler getNotificationHandler() {
172
		return notificationHandler;
173
	}
174

  
175
	/**
176
	 * @param notificationHandler the notificationHandler to set
177
	 */
178
	@Required
179
	public void setNotificationHandler(final NotificationHandler notificationHandler) {
180
		this.notificationHandler = notificationHandler;
181
	}
182

  
183
	@Override
184
	public List<String> listPlugins() throws DownloadServiceException {
185

  
186
		List<String> result = new ArrayList<String>();
187
		result.addAll(downloadPluginEnumerator.getAll().keySet());
188
		return result;
189
	}
190

  
191
}
modules/dnet-download-service/tags/dnet-download-service-2.2.1/src/main/resources/eu/dnetlib/data/download/applicationContext-dnet-downloadService.properties
1
services.download.http.sslcheck=false
2
services.download.https.protocols=TLSv1,TLSv1.1,TLSv1.2
modules/dnet-download-service/tags/dnet-download-service-2.2.1/src/main/resources/eu/dnetlib/data/download/applicationContext-dnet-downloadService.xml
1
<?xml version="1.0" encoding="UTF-8"?>
2
<beans xmlns="http://www.springframework.org/schema/beans"
3
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws"
4
	xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing"
5
	xmlns:p="http://www.springframework.org/schema/p" xmlns:http="http://cxf.apache.org/transports/http/configuration"
6
	xmlns:t="http://dnetlib.eu/springbeans/t" xmlns:template="http://dnetlib.eu/springbeans/template"
7
	xmlns:util="http://www.springframework.org/schema/util"
8
	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
9
                                    http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd
10
                                    http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd
11
                                    http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd
12
                            http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd
13
                            http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd
14
                            http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">
15
                            
16
    <bean id="downloadNotificationHandler"
17
		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerExecutorNotificationHandler"
18
		p:blackboardExecutor-ref="downloadBlackboardExecutor" />
19
		
20
	<bean id = "DownloadServiceFeeder" class ="eu.dnetlib.data.download.rmi.DownloadServiceFeeder" />
21

  
22
	<bean id="downloadBlackboardExecutor"
23
		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerActionExecutor"
24
		p:blackboardHandler-ref="blackboardHandler"
25
		p:actionType="eu.dnetlib.data.download.rmi.DownloadServiceActions"
26
		p:incomplete="false">
27
		<property name="actionMap">
28
			<map>
29
				<entry key="DOWNLOAD">
30
					<bean class="eu.dnetlib.data.download.DownloadAction"/>
31
				</entry>				
32
			</map>
33
		</property>
34
	</bean>		
35
                            
36
                            
37
    <bean id="DummyPlugin" class="eu.dnetlib.data.download.plugins.DummyPlugin" />
38

  
39
	<bean id="downloadSerive" class="eu.dnetlib.data.download.DownloadServiceImpl"
40
		init-method="start" p:notificationHandler-ref="downloadNotificationHandler" />
41

  
42
	<bean id="downloadPluginEnumerator" class="eu.dnetlib.data.download.DownloadPluginEnumeratorImpl" />
43

  
44
	<jaxws:endpoint id="downloadServiceEndpoint"
45
		implementor="#downloadSerive" implementorClass="eu.dnetlib.data.download.rmi.DownloadService"
46
		address="/downloadService" />
47

  
48
	<template:instance name="serviceRegistrationManager"
49
		t:serviceRegistrationManagerClass="eu.dnetlib.enabling.tools.registration.ValidatingServiceRegistrationManagerImpl"
50
		t:name="downloadServiceRegistrationManager" t:service="downloadSerive"
51
		t:endpoint="downloadServiceEndpoint" t:jobScheduler="jobScheduler"
52
		t:serviceRegistrator="blackboardServiceRegistrator" />
53

  
54

  
55
</beans>
modules/dnet-download-service/tags/dnet-download-service-2.2.1/src/main/resources/log4j.properties
1
log4j.rootLogger=WARN, CONSOLE
2
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
3
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
4
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
5

  
6
org.apache.cxf.Logger=org.apache.cxf.common.logging.Log4jLogger
7

  
8
### Application Level ###
9
log4j.logger.eu.dnetlib=INFO
10
log4j.logger.eu.dnetlib.data.download=DEBUG
11

  
12

  
13

  
modules/dnet-download-service/tags/dnet-download-service-2.2.1/pom.xml
1
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
2
	<parent>
3
		<groupId>eu.dnetlib</groupId>
4
		<artifactId>dnet45-parent</artifactId>
5
		<version>1.0.0</version>
6
		<relativePath />
7
	</parent>
8
	<modelVersion>4.0.0</modelVersion>
9
	<groupId>eu.dnetlib</groupId>
10
	<artifactId>dnet-download-service</artifactId>
11
	<version>2.2.1</version>
12
	<scm>
13
		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-download-service/tags/dnet-download-service-2.2.1</developerConnection>
14
	</scm>
15
	<dependencies>
16
		<dependency>
17
			<groupId>eu.dnetlib</groupId>
18
			<artifactId>dnet-download-service-rmi</artifactId>
19
			<version>[1.0.0,2.0.0)</version>
20
		</dependency>
21
		<dependency>
22
			<groupId>eu.dnetlib</groupId>
23
			<artifactId>cnr-service-common</artifactId>
24
			<version>[2.1.0,3.0.0)</version>
25
		</dependency>
26
		<dependency>
27
			<groupId>eu.dnetlib</groupId>
28
			<artifactId>dnet-modular-objectstore-service</artifactId>
29
			<version>[4.0.0,5.0.0)</version>
30
		</dependency>
31
		<dependency>
32
			<groupId>junit</groupId>
33
			<artifactId>junit</artifactId>
34
			<version>${junit.version}</version>
35
			<scope>test</scope>
36
		</dependency>
37
		<dependency>
38
			<groupId>org.mockito</groupId>
39
			<artifactId>mockito-core</artifactId>
40
			<version>${mockito.version}</version>
41
			<scope>test</scope>
42
		</dependency>
43
	</dependencies>
44
</project>

Also available in: Unified diff