Project

General

Profile

« Previous | Next » 

Revision 54375

Added by Claudio Atzori over 5 years ago

switched to org.apache.httpcomponents:httpclient:4.5.3

View differences:

HttpListIterator.java
1 1 
package eu.dnetlib.data.collector.plugins.httplist;
2 2 

  		




  
  3
  
    
import org.apache.commons.io.IOUtils;


  		




  
  4
  
    
import org.apache.commons.lang3.StringUtils;


  		




  
  5
  
    
import org.apache.http.HttpStatus;


  		




  
  6
  
    
import org.apache.http.client.methods.CloseableHttpResponse;


  		




  
  7
  
    
import org.apache.http.client.methods.HttpGet;


  		




  
  8
  
    
import org.apache.http.impl.client.CloseableHttpClient;


  		




  
  9
  
    
import org.apache.http.impl.client.HttpClients;


  		




  
  10
  
    

  		




  3
  11
  
    
import java.io.BufferedInputStream;


  		




  4
  12
  
    
import java.io.BufferedReader;


  		




  5
  13
  
    
import java.io.IOException;


  		




  6
  14
  
    
import java.io.StringReader;


  		




  7
  15
  
    
import java.util.Iterator;


  		




  8
  16
  
    

  		




  9
  
  
    
import eu.dnetlib.data.collector.plugins.HttpConnector;


  		




  10
  
  
    
import eu.dnetlib.data.collector.rmi.CollectorServiceException;


  		




  11
  
  
    
import org.apache.commons.io.IOUtils;


  		




  12
  
  
    
import org.apache.commons.lang3.StringUtils;


  		




  13
  
  
    

  		




  14
  17
  
    
public class HttpListIterator implements Iterator<String> {


  		




  15
  18
  
    

  		




  16
  
  
    
	private HttpConnector httpConnector;


  		




  
  19
  
    
	final CloseableHttpClient client = HttpClients.createDefault();


  		




  17
  20
  
    

  		




  18
  21
  
    
	private String baseUrl;


  		




  19
  22
  
    
	private String currentLine;


  		




  20
  23
  
    
	private BufferedReader reader;


  		




  21
  24
  
    

  		




  22
  
  
    
	public HttpListIterator(final String baseUrl, final String listAddress, final HttpConnector httpConnector) {


  		




  23
  
  
    
		this.httpConnector = httpConnector;


  		




  24
  
  
    
		this.baseUrl = baseUrl;


  		




  
  25
  
    
	public HttpListIterator(final String baseUrl, final String listAddress) {


  		




  25
  26
  
    
		try {


  		




  
  27
  
    
			this.baseUrl = baseUrl;


  		




  26
  28
  
    
			this.reader = new BufferedReader(new StringReader(download(listAddress)));


  		




  27
  29
  
    
			this.currentLine = reader.readLine();


  		




  28
  30
  
    
		} catch (Exception e) {


  		




  ......




  53
  55
  
    
	}


  		




  54
  56
  
    

  		




  55
  57
  
    
	private String download(final String url) {


  		




  56
  
  
    
		try {


  		




  57
  
  
    
			return IOUtils.toString(new BufferedInputStream(httpConnector.getInputSourceAsStream(url)));


  		




  58
  
  
    
		} catch(Exception e) {


  		




  59
  
  
    
			throw new RuntimeException(String.format("Error downloading url: %s, error: %s", url, e.getMessage()));


  		




  
  58
  
    
		try(CloseableHttpResponse response = client.execute(new HttpGet(url))) {


  		




  
  59
  
    

  		




  
  60
  
    
			final int responseCode = response.getStatusLine().getStatusCode();


  		




  
  61
  
    

  		




  
  62
  
    
			if (responseCode == HttpStatus.SC_OK) {


  		




  
  63
  
    
				return IOUtils.toString(new BufferedInputStream(response.getEntity().getContent()));


  		




  
  64
  
    
			} else {


  		




  
  65
  
    
				throw new RuntimeException("Error " + responseCode + " dowloading url: " + url);


  		




  
  66
  
    
			}


  		




  
  67
  
    

  		




  
  68
  
    
		} catch (IOException e) {


  		




  
  69
  
    
			throw new RuntimeException("Error dowloading url: " + url, e);


  		




  60
  70
  
    
		}


  		




  61
  71
  
    
	}


  		




  62
  72
  
    

  		




  63
  73
  
    
	@Override


  		




  64
  74
  
    
	public void remove() {}


  		




  65
  75
  
    

  		




  66
  
  
    
	public HttpConnector getHttpConnector() {


  		




  67
  
  
    
		return httpConnector;


  		




  68
  
  
    
	}


  		




  69
  
  
    

  		




  70
  
  
    
	public void setHttpConnector(final HttpConnector httpConnector) {


  		




  71
  
  
    
		this.httpConnector = httpConnector;


  		




  72
  
  
    
	}


  		




  73
  76
  
    
}


  		












Also available in:  Unified diff