Project

General

Profile

« Previous | Next » 

Revision 42936

Added by Claudio Atzori almost 8 years ago

adapted to org.apache.httpcomponents:httpclient:4.4.1

View differences:

modules/dnet-data-services/trunk/src/main/java/eu/dnetlib/data/collector/plugins/HttpCollectorPlugin.java
1 1 
package eu.dnetlib.data.collector.plugins;
2 2 

  		




  3
  3
  
    
import java.io.BufferedInputStream;


  		




  4
  
  
    

  		




  
  4
  
    
import java.io.IOException;


  		




  5
  5
  
    
import eu.dnetlib.rmi.data.CollectorServiceException;


  		




  6
  
  
    
import org.apache.commons.httpclient.HttpClient;


  		




  7
  
  
    
import org.apache.commons.httpclient.HttpMethod;


  		




  8
  
  
    
import org.apache.commons.httpclient.HttpStatus;


  		




  9
  
  
    
import org.apache.commons.httpclient.methods.GetMethod;


  		




  
  6
  
    
import org.apache.http.HttpStatus;


  		




  
  7
  
    
import org.apache.http.client.methods.CloseableHttpResponse;


  		




  
  8
  
    
import org.apache.http.client.methods.HttpGet;


  		




  
  9
  
    
import org.apache.http.impl.client.CloseableHttpClient;


  		




  
  10
  
    
import org.apache.http.impl.client.HttpClients;


  		




  10
  11
  
    

  		




  11
  12
  
    
public class HttpCollectorPlugin extends AbstractSplittedRecordPlugin {


  		




  12
  13
  
    

  		




  13
  14
  
    
	@Override


  		




  14
  15
  
    
	protected BufferedInputStream getBufferedInputStream(final String baseUrl) throws CollectorServiceException {


  		




  15
  
  
    
		try {


  		




  16
  
  
    
			final HttpClient client = new HttpClient();


  		




  17
  
  
    
			final HttpMethod method = new GetMethod(baseUrl);


  		




  18
  
  
    
			final int responseCode = client.executeMethod(method);


  		




  
  16
  
    
		try(CloseableHttpClient client = HttpClients.createDefault()) {


  		




  
  17
  
    
			final HttpGet method = new HttpGet(baseUrl);


  		




  
  18
  
    
			try(CloseableHttpResponse response = client.execute(method)) {


  		




  
  19
  
    
				int statusCode = response.getStatusLine().getStatusCode();


  		




  19
  20
  
    

  		




  20
  
  
    
			if (HttpStatus.SC_OK != responseCode) { throw new CollectorServiceException("Error " + responseCode + " dowloading url: " + baseUrl); }


  		




  21
  
  
    
			return new BufferedInputStream(method.getResponseBodyAsStream());


  		




  22
  
  
    
		} catch (Exception e) {


  		




  
  21
  
    
				if (HttpStatus.SC_OK != statusCode) {


  		




  
  22
  
    
					throw new CollectorServiceException("Error " + statusCode + " dowloading url: " + baseUrl);


  		




  
  23
  
    
				}


  		




  
  24
  
    

  		




  
  25
  
    
				return new BufferedInputStream(response.getEntity().getContent());


  		




  
  26
  
    
			}


  		




  
  27
  
    
		} catch (IOException e) {


  		




  23
  28
  
    
			throw new CollectorServiceException("Error dowloading url: " + baseUrl);


  		




  24
  29
  
    
		}


  		




  25
  30
  
    
	}


  		












  

    
      modules/dnet-data-services/trunk/src/main/java/eu/dnetlib/data/collector/plugins/httplist/HttpListIterator.java
    
  





  6
  6
  
    
import java.io.StringReader;


  		




  7
  7
  
    
import java.util.Iterator;


  		




  8
  8
  
    

  		




  9
  
  
    
import org.apache.commons.httpclient.HttpClient;


  		




  10
  
  
    
import org.apache.commons.httpclient.HttpMethod;


  		




  11
  
  
    
import org.apache.commons.httpclient.HttpStatus;


  		




  12
  
  
    
import org.apache.commons.httpclient.methods.GetMethod;


  		




  
  9
  
    
import eu.dnetlib.rmi.data.CollectorServiceRuntimeException;


  		




  13
  10
  
    
import org.apache.commons.io.IOUtils;


  		




  14
  11
  
    
import org.apache.commons.lang3.StringUtils;


  		




  
  12
  
    
import org.apache.http.HttpStatus;


  		




  
  13
  
    
import org.apache.http.client.methods.CloseableHttpResponse;


  		




  
  14
  
    
import org.apache.http.client.methods.HttpGet;


  		




  
  15
  
    
import org.apache.http.impl.client.HttpClients;


  		




  
  16
  
    
import org.apache.http.impl.client.CloseableHttpClient;


  		




  15
  17
  
    

  		




  16
  18
  
    
public class HttpListIterator implements Iterator<String> {


  		




  17
  19
  
    

  		




  18
  
  
    
	final HttpClient client = new HttpClient();


  		




  
  20
  
    
	private final CloseableHttpClient client = HttpClients.createDefault();


  		




  19
  21
  
    

  		




  20
  22
  
    
	private String baseUrl;


  		




  21
  23
  
    
	private String currentLine;


  		




  ......




  27
  29
  
    
			this.reader = new BufferedReader(new StringReader(download(listAddress)));


  		




  28
  30
  
    
			this.currentLine = reader.readLine();


  		




  29
  31
  
    
		} catch (Exception e) {


  		




  30
  
  
    
			throw new RuntimeException("Error creating iterator", e);


  		




  
  32
  
    
			throw new CollectorServiceRuntimeException("Error creating iterator", e);


  		




  31
  33
  
    
		}


  		




  32
  34
  
    
	}


  		




  33
  35
  
    

  		




  ......




  42
  44
  
    
			if (StringUtils.isNotBlank(currentLine)) {


  		




  43
  45
  
    
				return download(baseUrl + currentLine);


  		




  44
  46
  
    
			} else {


  		




  45
  
  
    
				throw new RuntimeException("Iterator has reached the end");


  		




  
  47
  
    
				throw new CollectorServiceRuntimeException("Iterator has reached the end");


  		




  46
  48
  
    
			}


  		




  47
  49
  
    
		} finally {


  		




  48
  50
  
    
			try {


  		




  49
  51
  
    
				this.currentLine = reader.readLine();


  		




  50
  52
  
    
			} catch (IOException e) {


  		




  51
  
  
    
				throw new RuntimeException("Error obtaining next element " + currentLine, e);


  		




  
  53
  
    
				throw new CollectorServiceRuntimeException("Error obtaining next element " + currentLine, e);


  		




  52
  54
  
    
			}


  		




  53
  55
  
    
		}


  		




  54
  56
  
    
	}


  		




  55
  57
  
    

  		




  56
  58
  
    
	private String download(final String url) {


  		




  57
  
  
    
		try {


  		




  58
  
  
    
			final HttpMethod method = new GetMethod(url);


  		




  59
  
  
    
			final int responseCode = client.executeMethod(method);


  		




  60
  
  
    
			if (responseCode == HttpStatus.SC_OK) {


  		




  61
  
  
    
				return IOUtils.toString(new BufferedInputStream(method.getResponseBodyAsStream()));


  		




  
  59
  
    
		final HttpGet method = new HttpGet(url);


  		




  
  60
  
    

  		




  
  61
  
    
		try(CloseableHttpResponse response = client.execute(method)) {


  		




  
  62
  
    
			int statusCode = response.getStatusLine().getStatusCode();


  		




  
  63
  
    

  		




  
  64
  
    
			if (HttpStatus.SC_OK == statusCode) {


  		




  
  65
  
    
				return IOUtils.toString(new BufferedInputStream(response.getEntity().getContent()));


  		




  62
  66
  
    
			} else {


  		




  63
  
  
    
				throw new RuntimeException("Error " + responseCode + " dowloading url: " + url);


  		




  
  67
  
    
				throw new CollectorServiceRuntimeException("Error " + statusCode + " dowloading url: " + url);


  		




  64
  68
  
    
			}


  		




  
  69
  
    

  		




  65
  70
  
    
		} catch (IOException e) {


  		




  66
  
  
    
			throw new RuntimeException("Error dowloading url: " + url);


  		




  
  71
  
    
			throw new CollectorServiceRuntimeException("Error downloading url: " + url);


  		




  67
  72
  
    
		}


  		




  68
  73
  
    
	}


  		




  69
  74
  
    

  		




  70
  75
  
    
	@Override


  		




  71
  76
  
    
	public void remove() {


  		




  
  77
  
    
		throw new UnsupportedOperationException();


  		




  72
  78
  
    
	}


  		




  73
  79
  
    

  		




  74
  80
  
    
}


  		












Also available in:  Unified diff