Revision 29020
Added by Michele Artini about 10 years ago
| modules/dnet-modular-collector-service/trunk/src/main/java/eu/dnetlib/data/collector/plugins/oai/engine/HttpConnector.java | ||
|---|---|---|
| 24 | 24 |
import eu.dnetlib.data.collector.rmi.CollectorServiceException; |
| 25 | 25 |
|
| 26 | 26 |
/** |
| 27 |
* @author jochen |
|
| 27 |
* @author jochen, michele
|
|
| 28 | 28 |
* |
| 29 | 29 |
*/ |
| 30 | 30 |
public class HttpConnector {
|
| 31 |
|
|
| 31 |
|
|
| 32 | 32 |
private static final Log log = LogFactory.getLog(HttpConnector.class); |
| 33 | 33 |
|
| 34 |
private static final int MAX_NUMBER_OF_RETRIES = 6; |
|
| 35 |
private static final int DEFAULT_DELAY = 60; //seconds |
|
| 36 |
|
|
| 34 | 37 |
/** |
| 35 | 38 |
* @param requestUrl |
| 36 | 39 |
* @return the content of the downloaded resource |
| 37 | 40 |
* @throws CollectorServiceException |
| 38 | 41 |
*/ |
| 39 | 42 |
public String getInputSource(final String requestUrl) throws CollectorServiceException {
|
| 43 |
return attemptDownload(requestUrl, 1); |
|
| 44 |
} |
|
| 40 | 45 |
|
| 41 |
log.debug("Downloading " + requestUrl);
|
|
| 42 |
|
|
| 46 |
private String attemptDownload(final String requestUrl, final int retryNumber) throws CollectorServiceException {
|
|
| 47 |
|
|
| 48 |
if (retryNumber > MAX_NUMBER_OF_RETRIES) {
|
|
| 49 |
throw new CollectorServiceException("Max number of retries exceeded");
|
|
| 50 |
} |
|
| 51 |
|
|
| 52 |
log.debug("Downloading " + requestUrl + " - try: " + retryNumber);
|
|
| 43 | 53 |
try {
|
| 44 |
final HttpURLConnection urlConn = (HttpURLConnection) (new URL(requestUrl)).openConnection(); |
|
| 54 |
try {
|
|
| 55 |
final HttpURLConnection urlConn = (HttpURLConnection) (new URL(requestUrl)).openConnection(); |
|
| 45 | 56 |
|
| 46 |
if (log.isDebugEnabled()) {
|
|
| 47 |
logHeaderFields(urlConn); |
|
| 57 |
if (log.isDebugEnabled()) {
|
|
| 58 |
logHeaderFields(urlConn); |
|
| 59 |
} |
|
| 60 |
|
|
| 61 |
int retryAfter = obtainRetryAfter(urlConn.getHeaderFields()); |
|
| 62 |
if (retryAfter > 0) {
|
|
| 63 |
log.warn("waiting and repeating request after " + retryAfter + " sec.");
|
|
| 64 |
Thread.sleep(retryAfter * 1000); |
|
| 65 |
return attemptDownload(requestUrl, retryNumber + 1); |
|
| 66 |
} else if (urlConn.getResponseCode() == HttpURLConnection.HTTP_MOVED_PERM || urlConn.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP){
|
|
| 67 |
final String newUrl = obtainNewLocation(urlConn.getHeaderFields()); |
|
| 68 |
log.info("The requested url has been moved to " + newUrl);
|
|
| 69 |
return attemptDownload(newUrl, retryNumber + 1); |
|
| 70 |
} else if ((urlConn.getResponseCode() != HttpURLConnection.HTTP_OK)) {
|
|
| 71 |
log.error("HTTP error: " + urlConn.getResponseMessage());
|
|
| 72 |
Thread.sleep(DEFAULT_DELAY * 1000); |
|
| 73 |
return attemptDownload(requestUrl, retryNumber + 1); |
|
| 74 |
} else {
|
|
| 75 |
return IOUtils.toString(urlConn.getInputStream()); |
|
| 76 |
} |
|
| 77 |
} catch (IOException e) {
|
|
| 78 |
log.error("error while retrieving from http-connection occured: " + e);
|
|
| 79 |
Thread.sleep(DEFAULT_DELAY * 1000); |
|
| 80 |
return attemptDownload(requestUrl, retryNumber + 1); |
|
| 48 | 81 |
} |
| 49 |
|
|
| 50 |
int retryAfter = obtainRetryAfter(urlConn.getHeaderFields()); |
|
| 51 |
if (retryAfter > 0) {
|
|
| 52 |
log.warn("waiting and repeating request after " + retryAfter + " sec.");
|
|
| 53 |
Thread.sleep(retryAfter * 1000); |
|
| 54 |
return getInputSource(requestUrl); |
|
| 55 |
} else if (urlConn.getResponseCode() == HttpURLConnection.HTTP_MOVED_PERM || urlConn.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP){
|
|
| 56 |
return getInputSource(obtainNewLocation(urlConn.getHeaderFields())); |
|
| 57 |
} else if ((urlConn.getResponseCode() != HttpURLConnection.HTTP_OK)) {
|
|
| 58 |
throw new CollectorServiceException(urlConn.getResponseMessage()); |
|
| 59 |
} else {
|
|
| 60 |
return IOUtils.toString(urlConn.getInputStream()); |
|
| 61 |
} |
|
| 62 |
} catch (IOException e) {
|
|
| 63 |
log.fatal("error while retrieving from http-connection occured: " + e);
|
|
| 64 |
throw new CollectorServiceException(e); |
|
| 65 | 82 |
} catch (InterruptedException e) {
|
| 66 |
log.fatal("Error in sleep: " + e);
|
|
| 83 |
log.error("Error in sleep: " + e);
|
|
| 67 | 84 |
throw new CollectorServiceException(e); |
| 68 | 85 |
} |
| 69 | 86 |
} |
| 70 | 87 |
|
| 71 | 88 |
private void logHeaderFields(final HttpURLConnection urlConn) throws IOException {
|
| 72 | 89 |
log.debug("StatusCode: " + urlConn.getResponseMessage());
|
| 73 |
|
|
| 90 |
|
|
| 74 | 91 |
for (Map.Entry<String, List<String>> e : urlConn.getHeaderFields().entrySet()){
|
| 75 | 92 |
if (e.getKey() != null) {
|
| 76 | 93 |
for (String v : e.getValue()) {
|
| 77 |
log.debug(" key: " + e.getKey() + ", value: " + v);
|
|
| 94 |
log.debug(" key: " + e.getKey() + " - value: " + v);
|
|
| 78 | 95 |
} |
| 79 | 96 |
} |
| 80 | 97 |
} |
| 81 | 98 |
} |
| 82 |
|
|
| 99 |
|
|
| 83 | 100 |
private int obtainRetryAfter(final Map<String, List<String>> headerMap) {
|
| 84 | 101 |
for (String key : headerMap.keySet()){
|
| 85 | 102 |
if (key != null && key.toLowerCase().equals("retry-after") && headerMap.get(key).size() > 0 && NumberUtils.isNumber(headerMap.get(key).get(0))) {
|
| ... | ... | |
| 88 | 105 |
} |
| 89 | 106 |
return -1; |
| 90 | 107 |
} |
| 91 |
|
|
| 108 |
|
|
| 92 | 109 |
private String obtainNewLocation(final Map<String, List<String>> headerMap) throws CollectorServiceException {
|
| 93 | 110 |
for (String key : headerMap.keySet()){
|
| 94 | 111 |
if (key != null && key.toLowerCase().equals("location") && headerMap.get(key).size() > 0) {
|
Also available in: Unified diff
retry on exceptions