| 22 |
22 |
import org.apache.commons.logging.Log;
|
| 23 |
23 |
import org.apache.commons.logging.LogFactory;
|
| 24 |
24 |
|
|
25 |
import eu.dnetlib.data.collector.plugin.CollectorPluginErrorLogList;
|
| 25 |
26 |
import eu.dnetlib.data.collector.rmi.CollectorServiceException;
|
| 26 |
27 |
|
| 27 |
28 |
/**
|
| ... | ... | |
| 42 |
43 |
* @throws CollectorServiceException
|
| 43 |
44 |
*/
|
| 44 |
45 |
public String getInputSource(final String requestUrl) throws CollectorServiceException {
|
| 45 |
|
return attemptDownload(requestUrl, 1, "n/a");
|
|
46 |
return attemptDownload(requestUrl, 1, new CollectorPluginErrorLogList());
|
| 46 |
47 |
}
|
| 47 |
48 |
|
| 48 |
|
private String attemptDownload(final String requestUrl, final int retryNumber, final String lastErrorHelperText) throws CollectorServiceException {
|
|
49 |
private String attemptDownload(final String requestUrl, final int retryNumber, final CollectorPluginErrorLogList errorList)
|
|
50 |
throws CollectorServiceException {
|
| 49 |
51 |
|
| 50 |
|
if (retryNumber > maxNumberOfRetry) { throw new CollectorServiceException("Max number of retries exceeded. Cause: " + lastErrorHelperText); }
|
|
52 |
if (retryNumber > maxNumberOfRetry) { throw new CollectorServiceException("Max number of retries exceeded. Cause: \n " + errorList); }
|
| 51 |
53 |
|
| 52 |
54 |
log.debug("Downloading " + requestUrl + " - try: " + retryNumber);
|
| 53 |
55 |
try {
|
| ... | ... | |
| 66 |
68 |
if (retryAfter > 0) {
|
| 67 |
69 |
log.warn("waiting and repeating request after " + retryAfter + " sec.");
|
| 68 |
70 |
Thread.sleep(retryAfter * 1000);
|
| 69 |
|
return attemptDownload(requestUrl, retryNumber + 1, "n/a");
|
|
71 |
errorList.add("503 Service Unavailable");
|
|
72 |
return attemptDownload(requestUrl, retryNumber + 1, errorList);
|
| 70 |
73 |
} else if ((urlConn.getResponseCode() == HttpURLConnection.HTTP_MOVED_PERM) || (urlConn.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP)) {
|
| 71 |
74 |
final String newUrl = obtainNewLocation(urlConn.getHeaderFields());
|
| 72 |
75 |
log.info("The requested url has been moved to " + newUrl);
|
| 73 |
|
return attemptDownload(newUrl, retryNumber + 1, String.format("%s %s", urlConn.getResponseCode(), urlConn.getResponseMessage()));
|
|
76 |
errorList.add(String.format("%s %s. Moved to: %s", urlConn.getResponseCode(), urlConn.getResponseMessage(), newUrl));
|
|
77 |
return attemptDownload(newUrl, retryNumber + 1, errorList);
|
| 74 |
78 |
} else if (urlConn.getResponseCode() != HttpURLConnection.HTTP_OK) {
|
| 75 |
79 |
log.error(String.format("HTTP error: %s %s", urlConn.getResponseCode(), urlConn.getResponseMessage()));
|
| 76 |
80 |
Thread.sleep(defaultDelay * 1000);
|
| 77 |
|
return attemptDownload(requestUrl, retryNumber + 1, String.format("%s %s", urlConn.getResponseCode(), urlConn.getResponseMessage()));
|
|
81 |
errorList.add(String.format("%s %s", urlConn.getResponseCode(), urlConn.getResponseMessage()));
|
|
82 |
return attemptDownload(requestUrl, retryNumber + 1, errorList);
|
| 78 |
83 |
} else {
|
| 79 |
84 |
input = urlConn.getInputStream();
|
| 80 |
85 |
return IOUtils.toString(input);
|
| ... | ... | |
| 82 |
87 |
} catch (IOException e) {
|
| 83 |
88 |
log.error("error while retrieving from http-connection occured: " + e, e);
|
| 84 |
89 |
Thread.sleep(defaultDelay * 1000);
|
| 85 |
|
return attemptDownload(requestUrl, retryNumber + 1, e.getMessage());
|
|
90 |
errorList.add(e.getMessage());
|
|
91 |
return attemptDownload(requestUrl, retryNumber + 1, errorList);
|
| 86 |
92 |
} finally {
|
| 87 |
93 |
IOUtils.closeQuietly(input);
|
| 88 |
94 |
}
|
Added by Andrea Mannocci over 9 years ago
solving ticket #1160
Created a collectorServiceErrorMessageLogList to help jochen in debugging OAI harvests gone wrong.