Revision 57866
Added by Alessia Bardi over 4 years ago
modules/dnet-modular-collector-service/trunk/src/main/java/eu/dnetlib/data/collector/plugins/HttpConnector.java | ||
---|---|---|
20 | 20 |
import org.apache.commons.logging.LogFactory; |
21 | 21 |
|
22 | 22 |
/** |
23 |
* @author jochen, michele, andrea |
|
23 |
* @author jochen, michele, andrea, alessia
|
|
24 | 24 |
*/ |
25 | 25 |
public class HttpConnector { |
26 | 26 |
|
... | ... | |
100 | 100 |
} |
101 | 101 |
|
102 | 102 |
int retryAfter = obtainRetryAfter(urlConn.getHeaderFields()); |
103 |
if (retryAfter > 0 && urlConn.getResponseCode() == HttpURLConnection.HTTP_UNAVAILABLE) { |
|
104 |
log.warn("waiting and repeating request after " + retryAfter + " sec."); |
|
105 |
Thread.sleep(retryAfter * 1000); |
|
106 |
errorList.add("503 Service Unavailable"); |
|
107 |
urlConn.disconnect(); |
|
108 |
return attemptDownload(requestUrl, retryNumber + 1, errorList); |
|
109 |
} else if ((urlConn.getResponseCode() == HttpURLConnection.HTTP_MOVED_PERM) || (urlConn.getResponseCode() |
|
110 |
== HttpURLConnection.HTTP_MOVED_TEMP)) { |
|
111 |
final String newUrl = obtainNewLocation(urlConn.getHeaderFields()); |
|
112 |
log.debug("The requested url has been moved to " + newUrl); |
|
113 |
errorList.add(String.format("%s %s. Moved to: %s", urlConn.getResponseCode(), urlConn.getResponseMessage(), newUrl)); |
|
114 |
urlConn.disconnect(); |
|
115 |
return attemptDownload(newUrl, retryNumber + 1, errorList); |
|
116 |
} else if (urlConn.getResponseCode() != HttpURLConnection.HTTP_OK) { |
|
117 |
log.error(String.format("HTTP error: %s %s", urlConn.getResponseCode(), urlConn.getResponseMessage())); |
|
118 |
Thread.sleep(defaultDelay * 1000); |
|
119 |
errorList.add(String.format("%s %s", urlConn.getResponseCode(), urlConn.getResponseMessage())); |
|
120 |
urlConn.disconnect(); |
|
121 |
return attemptDownload(requestUrl, retryNumber + 1, errorList); |
|
122 |
} else { |
|
103 |
if (is2xx(urlConn.getResponseCode())) { |
|
123 | 104 |
input = urlConn.getInputStream(); |
124 | 105 |
responseType = urlConn.getContentType(); |
125 | 106 |
return input; |
126 | 107 |
} |
108 |
if (is3xx(urlConn.getResponseCode())) { |
|
109 |
//REDIRECTS |
|
110 |
final String newUrl = obtainNewLocation(urlConn.getHeaderFields()); |
|
111 |
log.debug(String.format("The requested url %s has been moved to %s", requestUrl, newUrl)); |
|
112 |
errorList.add(String.format("%s %s %s. Moved to: %s", requestUrl, urlConn.getResponseCode(), urlConn.getResponseMessage(), newUrl)); |
|
113 |
urlConn.disconnect(); |
|
114 |
if (retryAfter > 0) Thread.sleep(retryAfter * 1000); |
|
115 |
return attemptDownload(newUrl, retryNumber + 1, errorList); |
|
116 |
} |
|
117 |
if (is4xx(urlConn.getResponseCode())) { |
|
118 |
//CLIENT ERROR, DO NOT RETRY |
|
119 |
errorList.add(String.format("%s error %s: %s", requestUrl, urlConn.getResponseCode(), urlConn.getResponseMessage())); |
|
120 |
throw new CollectorServiceException("4xx error: request will not be repeated." + errorList); |
|
121 |
} |
|
122 |
if (is5xx(urlConn.getResponseCode())) { |
|
123 |
//SERVER SIDE ERRORS RETRY ONLY on 503 |
|
124 |
switch (urlConn.getResponseCode()) { |
|
125 |
case HttpURLConnection.HTTP_UNAVAILABLE: |
|
126 |
if (retryAfter > 0) { |
|
127 |
log.warn(requestUrl+" - waiting and repeating request after suggested retry-after" + retryAfter + " sec."); |
|
128 |
Thread.sleep(retryAfter * 1000); |
|
129 |
} |
|
130 |
else { |
|
131 |
log.warn(requestUrl+" - waiting and repeating request after default delay of " + defaultDelay + " sec."); |
|
132 |
Thread.sleep(defaultDelay * 1000); |
|
133 |
} |
|
134 |
errorList.add(requestUrl+" 503 Service Unavailable"); |
|
135 |
urlConn.disconnect(); |
|
136 |
return attemptDownload(requestUrl, retryNumber + 1, errorList); |
|
137 |
default: |
|
138 |
errorList.add(String.format("%s Error %s: %s", requestUrl, urlConn.getResponseCode(), urlConn.getResponseMessage())); |
|
139 |
throw new CollectorServiceException(urlConn.getResponseCode() + "error" + errorList); |
|
140 |
} |
|
141 |
} |
|
142 |
throw new CollectorServiceException("Unexpected status code: " + urlConn.getResponseCode() + "error" + errorList); |
|
143 |
}catch(MalformedURLException | NoRouteToHostException e){ |
|
144 |
errorList.add(String.format("Error: %s for request url: %s", e.getCause(), requestUrl)); |
|
145 |
throw new CollectorServiceException(e+ "error "+errorList); |
|
127 | 146 |
} catch (IOException e) { |
128 |
log.error("error while retrieving from http-connection occured: " + requestUrl, e); |
|
129 | 147 |
Thread.sleep(defaultDelay * 1000); |
130 |
errorList.add(e.getMessage()); |
|
148 |
errorList.add(requestUrl+ " "+e.getMessage());
|
|
131 | 149 |
return attemptDownload(requestUrl, retryNumber + 1, errorList); |
132 | 150 |
} |
133 | 151 |
} catch (InterruptedException e) { |
... | ... | |
193 | 211 |
} |
194 | 212 |
} |
195 | 213 |
|
214 |
private boolean is2xx(final int statusCode) { |
|
215 |
return statusCode >= 200 && statusCode <=299; |
|
216 |
} |
|
217 |
|
|
218 |
private boolean is4xx(final int statusCode) { |
|
219 |
return statusCode >= 400 && statusCode <=499; |
|
220 |
} |
|
221 |
|
|
222 |
private boolean is3xx(final int statusCode) { |
|
223 |
return statusCode >= 300 && statusCode <=399; |
|
224 |
} |
|
225 |
|
|
226 |
private boolean is5xx(final int statusCode) { |
|
227 |
return statusCode >= 500 && statusCode <=599; |
|
228 |
} |
|
229 |
|
|
230 |
|
|
196 | 231 |
public int getMaxNumberOfRetry() { |
197 | 232 |
return maxNumberOfRetry; |
198 | 233 |
} |
Also available in: Unified diff
#5185: Improved logging and handling of HTTP status codes