Revision 53932
Added by Alessia Bardi over 5 years ago
HttpListIterator.java | ||
---|---|---|
6 | 6 |
import java.io.StringReader; |
7 | 7 |
import java.util.Iterator; |
8 | 8 |
|
9 |
import org.apache.commons.httpclient.HttpClient; |
|
10 |
import org.apache.commons.httpclient.HttpMethod; |
|
11 |
import org.apache.commons.httpclient.HttpStatus; |
|
12 |
import org.apache.commons.httpclient.methods.GetMethod; |
|
9 |
import eu.dnetlib.data.collector.plugins.HttpConnector; |
|
10 |
import eu.dnetlib.data.collector.rmi.CollectorServiceException; |
|
13 | 11 |
import org.apache.commons.io.IOUtils; |
14 | 12 |
import org.apache.commons.lang3.StringUtils; |
15 | 13 |
|
16 | 14 |
public class HttpListIterator implements Iterator<String> { |
17 | 15 |
|
18 |
final HttpClient client = new HttpClient();
|
|
16 |
private HttpConnector httpConnector;
|
|
19 | 17 |
|
20 | 18 |
private String baseUrl; |
21 | 19 |
private String currentLine; |
22 | 20 |
private BufferedReader reader; |
23 | 21 |
|
24 |
public HttpListIterator(final String baseUrl, final String listAddress) { |
|
22 |
public HttpListIterator(final String baseUrl, final String listAddress, final HttpConnector httpConnector) { |
|
23 |
this.httpConnector = httpConnector; |
|
24 |
this.baseUrl = baseUrl; |
|
25 | 25 |
try { |
26 |
this.baseUrl = baseUrl; |
|
27 | 26 |
this.reader = new BufferedReader(new StringReader(download(listAddress))); |
28 | 27 |
this.currentLine = reader.readLine(); |
29 | 28 |
} catch (Exception e) { |
... | ... | |
55 | 54 |
|
56 | 55 |
private String download(final String url) { |
57 | 56 |
try { |
58 |
final HttpMethod method = new GetMethod(url); |
|
59 |
final int responseCode = client.executeMethod(method); |
|
60 |
if (responseCode == HttpStatus.SC_OK) { |
|
61 |
return IOUtils.toString(new BufferedInputStream(method.getResponseBodyAsStream())); |
|
62 |
} else { |
|
63 |
throw new RuntimeException("Error " + responseCode + " dowloading url: " + url); |
|
64 |
} |
|
65 |
} catch (IOException e) { |
|
66 |
throw new RuntimeException("Error dowloading url: " + url); |
|
57 |
return IOUtils.toString(new BufferedInputStream(httpConnector.getInputSourceAsStream(url))); |
|
58 |
} catch(Exception e) { |
|
59 |
throw new RuntimeException(String.format("Error downloading url: %s, error: %s", url, e.getMessage())); |
|
67 | 60 |
} |
68 | 61 |
} |
69 | 62 |
|
70 | 63 |
@Override |
71 | 64 |
public void remove() {} |
72 | 65 |
|
66 |
public HttpConnector getHttpConnector() { |
|
67 |
return httpConnector; |
|
68 |
} |
|
69 |
|
|
70 |
public void setHttpConnector(final HttpConnector httpConnector) { |
|
71 |
this.httpConnector = httpConnector; |
|
72 |
} |
|
73 | 73 |
} |
Also available in: Unified diff
Upgrade to commons.net:commons.net 3.6 (from 3.3) and commons-httpclient 4.5.6 (from 3.1).
using HttpConnector where needed.
httpConnector can now return an InputStream and not only a string.