Project

General

Profile

« Previous | Next » 

Revision 57866

#5185: Improved logging and handling of HTTP status codes

View differences:

modules/dnet-modular-collector-service/trunk/src/main/java/eu/dnetlib/data/collector/plugins/HttpConnector.java
20 20
import org.apache.commons.logging.LogFactory;
21 21

  
22 22
/**
23
 * @author jochen, michele, andrea
23
 * @author jochen, michele, andrea, alessia
24 24
 */
25 25
public class HttpConnector {
26 26

  
......
100 100
				}
101 101

  
102 102
				int retryAfter = obtainRetryAfter(urlConn.getHeaderFields());
103
				if (retryAfter > 0 && urlConn.getResponseCode() == HttpURLConnection.HTTP_UNAVAILABLE) {
104
					log.warn("waiting and repeating request after " + retryAfter + " sec.");
105
					Thread.sleep(retryAfter * 1000);
106
					errorList.add("503 Service Unavailable");
107
					urlConn.disconnect();
108
					return attemptDownload(requestUrl, retryNumber + 1, errorList);
109
				} else if ((urlConn.getResponseCode() == HttpURLConnection.HTTP_MOVED_PERM) || (urlConn.getResponseCode()
110
						== HttpURLConnection.HTTP_MOVED_TEMP)) {
111
					final String newUrl = obtainNewLocation(urlConn.getHeaderFields());
112
					log.debug("The requested url has been moved to " + newUrl);
113
					errorList.add(String.format("%s %s. Moved to: %s", urlConn.getResponseCode(), urlConn.getResponseMessage(), newUrl));
114
					urlConn.disconnect();
115
					return attemptDownload(newUrl, retryNumber + 1, errorList);
116
				} else if (urlConn.getResponseCode() != HttpURLConnection.HTTP_OK) {
117
					log.error(String.format("HTTP error: %s %s", urlConn.getResponseCode(), urlConn.getResponseMessage()));
118
					Thread.sleep(defaultDelay * 1000);
119
					errorList.add(String.format("%s %s", urlConn.getResponseCode(), urlConn.getResponseMessage()));
120
					urlConn.disconnect();
121
					return attemptDownload(requestUrl, retryNumber + 1, errorList);
122
				} else {
103
				if (is2xx(urlConn.getResponseCode())) {
123 104
					input = urlConn.getInputStream();
124 105
					responseType = urlConn.getContentType();
125 106
					return input;
126 107
				}
108
				if (is3xx(urlConn.getResponseCode())) {
109
					//REDIRECTS
110
					final String newUrl = obtainNewLocation(urlConn.getHeaderFields());
111
					log.debug(String.format("The requested url %s has been moved to %s", requestUrl, newUrl));
112
					errorList.add(String.format("%s %s %s. Moved to: %s", requestUrl, urlConn.getResponseCode(), urlConn.getResponseMessage(), newUrl));
113
					urlConn.disconnect();
114
					if (retryAfter > 0) Thread.sleep(retryAfter * 1000);
115
					return attemptDownload(newUrl, retryNumber + 1, errorList);
116
				}
117
				if (is4xx(urlConn.getResponseCode())) {
118
					//CLIENT ERROR, DO NOT RETRY
119
					errorList.add(String.format("%s error %s: %s", requestUrl, urlConn.getResponseCode(), urlConn.getResponseMessage()));
120
					throw new CollectorServiceException("4xx error: request will not be repeated." + errorList);
121
				}
122
				if (is5xx(urlConn.getResponseCode())) {
123
					//SERVER SIDE ERRORS RETRY ONLY on 503
124
					switch (urlConn.getResponseCode()) {
125
						case HttpURLConnection.HTTP_UNAVAILABLE:
126
							if (retryAfter > 0) {
127
								log.warn(requestUrl+" - waiting and repeating request after suggested retry-after" + retryAfter + " sec.");
128
								Thread.sleep(retryAfter * 1000);
129
							}
130
							else {
131
								log.warn(requestUrl+" - waiting and repeating request after default delay of " + defaultDelay + " sec.");
132
								Thread.sleep(defaultDelay * 1000);
133
							}
134
							errorList.add(requestUrl+" 503 Service Unavailable");
135
							urlConn.disconnect();
136
							return attemptDownload(requestUrl, retryNumber + 1, errorList);
137
						default:
138
							errorList.add(String.format("%s Error %s: %s", requestUrl, urlConn.getResponseCode(), urlConn.getResponseMessage()));
139
							throw new CollectorServiceException(urlConn.getResponseCode() + "error" + errorList);
140
					}
141
				}
142
				throw new CollectorServiceException("Unexpected status code: " + urlConn.getResponseCode() + "error" + errorList);
143
			}catch(MalformedURLException | NoRouteToHostException e){
144
				errorList.add(String.format("Error: %s for request url: %s", e.getCause(), requestUrl));
145
				throw new CollectorServiceException(e+ "error "+errorList);
127 146
			} catch (IOException e) {
128
				log.error("error while retrieving from http-connection occured: " + requestUrl, e);
129 147
				Thread.sleep(defaultDelay * 1000);
130
				errorList.add(e.getMessage());
148
				errorList.add(requestUrl+ " "+e.getMessage());
131 149
				return attemptDownload(requestUrl, retryNumber + 1, errorList);
132 150
			}
133 151
		} catch (InterruptedException e) {
......
193 211
		}
194 212
	}
195 213

  
214
	private boolean is2xx(final int statusCode) {
215
		return statusCode >= 200 && statusCode <=299;
216
	}
217

  
218
	private boolean is4xx(final int statusCode) {
219
		return statusCode >= 400 && statusCode <=499;
220
	}
221

  
222
	private boolean is3xx(final int statusCode) {
223
		return statusCode >= 300 && statusCode <=399;
224
	}
225

  
226
	private boolean is5xx(final int statusCode) {
227
		return statusCode >= 500 && statusCode <=599;
228
	}
229

  
230

  
196 231
	public int getMaxNumberOfRetry() {
197 232
		return maxNumberOfRetry;
198 233
	}

Also available in: Unified diff