Project

General

Profile

« Previous | Next » 

Revision 54375

Added by Claudio Atzori over 5 years ago

switched to org.apache.httpcomponents:httpclient:4.5.3

View differences:

modules/dnet-modular-collector-service/trunk/src/test/java/eu/dnetlib/data/collector/plugins/HttpConnectorTest.java
1 1 
package eu.dnetlib.data.collector.plugins;
2 2 

  		




  3
  3
  
    
import java.io.IOException;


  		




  
  4
  
    
import java.io.InputStream;


  		




  4
  5
  
    
import java.net.HttpURLConnection;


  		




  5
  6
  
    
import java.net.URL;


  		




  
  7
  
    
import java.util.concurrent.TimeUnit;


  		




  
  8
  
    
import javax.net.ssl.HttpsURLConnection;


  		




  
  9
  
    
import javax.net.ssl.SSLProtocolException;


  		




  6
  10
  
    

  		




  
  11
  
    
import com.google.common.base.Joiner;


  		




  7
  12
  
    
import eu.dnetlib.data.collector.rmi.CollectorServiceException;


  		




  
  13
  
    
import org.apache.commons.io.IOUtils;


  		




  8
  14
  
    
import org.apache.commons.logging.Log;


  		




  9
  15
  
    
import org.apache.commons.logging.LogFactory;


  		




  
  16
  
    
import org.apache.http.HttpResponse;


  		




  
  17
  
    
import org.apache.http.client.methods.CloseableHttpResponse;


  		




  
  18
  
    
import org.apache.http.client.methods.HttpGet;


  		




  
  19
  
    
import org.apache.http.client.methods.HttpUriRequest;


  		




  
  20
  
    
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;


  		




  
  21
  
    
import org.apache.http.impl.client.HttpClientBuilder;


  		




  
  22
  
    
import org.apache.http.impl.client.HttpClients;


  		




  
  23
  
    
import org.apache.http.ssl.SSLContextBuilder;


  		




  10
  24
  
    
import org.junit.Before;


  		




  11
  25
  
    
import org.junit.Ignore;


  		




  12
  26
  
    
import org.junit.Test;


  		




  13
  
  
    

  		




  
  27
  
    
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;


  		




  14
  28
  
    
@Ignore


  		




  15
  29
  
    
public class HttpConnectorTest {


  		




  16
  30
  
    

  		




  ......




  21
  35
  
    
	private static final String URL_MISCONFIGURED_SERVER = "https://www.alexandria.unisg.ch/cgi/oai2?verb=Identify";


  		




  22
  36
  
    
	private static final String URL_GOODSNI_SERVER = "https://air.unimi.it/oai/openaire?verb=Identify";


  		




  23
  37
  
    

  		




  
  38
  
    
	private static final SSLContextBuilder sslContextBuilder = new SSLContextBuilder();


  		




  
  39
  
    
	private static SSLConnectionSocketFactory sslSocketFactory;


  		




  
  40
  
    

  		




  
  41
  
    
	private static final HttpComponentsClientHttpRequestFactory httpRequestFactory = new HttpComponentsClientHttpRequestFactory(HttpClientBuilder


  		




  
  42
  
    
			.create()


  		




  
  43
  
    
			.setConnectionTimeToLive(0, TimeUnit.MILLISECONDS)


  		




  
  44
  
    
			.setMaxConnPerRoute(1)


  		




  
  45
  
    
			.setMaxConnTotal(1)


  		




  
  46
  
    
			.disableAutomaticRetries()


  		




  
  47
  
    
			.disableConnectionState()


  		




  
  48
  
    
			.setSSLSocketFactory(sslSocketFactory)


  		




  
  49
  
    
			.build());


  		




  
  50
  
    

  		




  
  51
  
    
//	static {


  		




  
  52
  
    
//		System.setProperty("javax.net.debug", "ssl,handshake");


  		




  
  53
  
    
//		System.setProperty("jsse.enableSNIExtension", "true");


  		




  
  54
  
    
//		try {


  		




  
  55
  
    
//			sslContextBuilder.loadTrustMaterial(null, (chain, authType) -> true);


  		




  
  56
  
    
//			SSLParameters sslParameters = new SSLParameters();


  		




  
  57
  
    
//			List sniHostNames = new ArrayList(1);


  		




  
  58
  
    
//			//sniHostNames.add(new SNIHostName(url.getHost()));


  		




  
  59
  
    
//			sslParameters.setServerNames(sniHostNames);


  		




  
  60
  
    
//			sslSocketFactory = new SSLConnectionSocketFactory(sslContextBuilder.build().se, sslParameters);


  		




  
  61
  
    
//


  		




  
  62
  
    
//		} catch (final NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) {


  		




  
  63
  
    
//			log.error(e);;


  		




  
  64
  
    
//		}


  		




  
  65
  
    
//	}


  		




  
  66
  
    

  		




  24
  67
  
    
	@Before


  		




  25
  68
  
    
	public void setUp() {


  		




  26
  69
  
    
		connector = new HttpConnector();


  		




  ......




  28
  71
  
    

  		




  29
  72
  
    
	@Test


  		




  30
  73
  
    
	@Ignore


  		




  
  74
  
    
	public void testConnectionRelease() throws IOException, InterruptedException {


  		




  
  75
  
    

  		




  
  76
  
    
		InputStream in;


  		




  
  77
  
    

  		




  
  78
  
    
		final HttpGet get = new HttpGet("http://www.google.com");


  		




  
  79
  
    
		try(CloseableHttpResponse rsp = HttpClients.createDefault().execute(get)) {


  		




  
  80
  
    

  		




  
  81
  
    
			in = rsp.getEntity().getContent();


  		




  
  82
  
    
		}


  		




  
  83
  
    

  		




  
  84
  
    
		log.info("going to sleep ... ");


  		




  
  85
  
    
		Thread.sleep(1000);


  		




  
  86
  
    

  		




  
  87
  
    
		log.info("wake up!");


  		




  
  88
  
    

  		




  
  89
  
    
		System.out.println(IOUtils.toString(in));


  		




  
  90
  
    
	}


  		




  
  91
  
    

  		




  
  92
  
    
	@Test


  		




  
  93
  
    
	@Ignore


  		




  31
  94
  
    
	public void testGetInputSource() throws CollectorServiceException {


  		




  32
  95
  
    
		System.out.println(connector.getInputSource(URL));


  		




  33
  96
  
    
	}


  		




  ......




  38
  101
  
    
		System.out.println(connector.getInputSource(URL_MISCONFIGURED_SERVER));


  		




  39
  102
  
    
	}


  		




  40
  103
  
    

  		




  41
  
  
    
	@Test(expected = javax.net.ssl.SSLProtocolException.class)


  		




  
  104
  
    
	@Test


  		




  
  105
  
    
	@Ignore


  		




  42
  106
  
    
	public void testMisconfiguredServers2() throws IOException {


  		




  43
  107
  
    
		HttpURLConnection urlConn = (HttpURLConnection) new URL(URL_MISCONFIGURED_SERVER).openConnection();


  		




  44
  108
  
    
		urlConn.getResponseMessage();


  		




  45
  109
  
    
	}


  		




  46
  110
  
    

  		




  47
  111
  
    
	@Test


  		




  
  112
  
    
	public void testDisablingSNI() throws IOException {


  		




  
  113
  
    
		HttpURLConnection urlConn = null;


  		




  
  114
  
    
		try {


  		




  
  115
  
    
			urlConn = (HttpURLConnection) new URL(URL_MISCONFIGURED_SERVER).openConnection();


  		




  
  116
  
    
			urlConn.getResponseMessage();


  		




  
  117
  
    
		 } catch(SSLProtocolException sslExce) {


  		




  
  118
  
    
			if (sslExce.getMessage() != null && sslExce.getMessage().equals("handshake alert:  unrecognized_name")) {


  		




  
  119
  
    
				System.out.println("Server has misconfigured SSL SNI (handshake alert:  unrecognized_name). Trying to disable SNI");


  		




  
  120
  
    
				if (urlConn instanceof HttpsURLConnection) {


  		




  
  121
  
    
					HttpResponse res = httpRequestFactory.getHttpClient().execute(new HttpGet(URL_MISCONFIGURED_SERVER));


  		




  
  122
  
    
					System.out.println(res.getStatusLine());


  		




  
  123
  
    
//					HttpsURLConnection httpsUrlConnection = (HttpsURLConnection) urlConn;


  		




  
  124
  
    
//					httpsUrlConnection.setSSLSocketFactory(sslSocketFactory);


  		




  
  125
  
    
//					httpsUrlConnection.setHostnameVerifier(new HostnameVerifier() {


  		




  
  126
  
    
//						public boolean verify( String s, SSLSession sess ) {


  		




  
  127
  
    
//							return true;


  		




  
  128
  
    
//						}});


  		




  
  129
  
    
//					httpsUrlConnection.getResponseMessage();


  		




  
  130
  
    
				}


  		




  
  131
  
    

  		




  
  132
  
    
			}


  		




  
  133
  
    
		}


  		




  
  134
  
    
	}


  		




  
  135
  
    

  		




  
  136
  
    

  		




  
  137
  
    

  		




  
  138
  
    

  		




  
  139
  
    
	@Test


  		




  48
  140
  
    
	public void testGoodServers() throws CollectorServiceException {


  		




  49
  141
  
    
		System.out.println(connector.getInputSource(URL_GOODSNI_SERVER));


  		




  50
  142
  
    
	}


  		












  

    
      modules/dnet-modular-collector-service/trunk/src/main/java/eu/dnetlib/data/collector/plugins/HttpCollectorPlugin.java
    
  





  1
  1
  
    
package eu.dnetlib.data.collector.plugins;


  		




  2
  2
  
    

  		




  3
  
  
    
import java.io.BufferedInputStream;


  		




  4
  
  
    

  		




  5
  3
  
    
import eu.dnetlib.data.collector.rmi.CollectorServiceException;


  		




  6
  4
  
    
import org.apache.commons.io.IOUtils;


  		




  7
  
  
    
import org.springframework.beans.factory.annotation.Autowired;


  		




  
  5
  
    
import org.apache.http.HttpStatus;


  		




  
  6
  
    
import org.apache.http.client.methods.CloseableHttpResponse;


  		




  
  7
  
    
import org.apache.http.client.methods.HttpGet;


  		




  
  8
  
    
import org.apache.http.impl.client.HttpClients;


  		




  8
  9
  
    

  		




  
  10
  
    
import java.io.BufferedInputStream;


  		




  
  11
  
    
import java.io.ByteArrayInputStream;


  		




  
  12
  
    
import java.io.IOException;


  		




  
  13
  
    
import java.io.InputStream;


  		




  
  14
  
    

  		




  9
  15
  
    
public class HttpCollectorPlugin extends AbstractSplittedRecordPlugin {


  		




  10
  16
  
    

  		




  11
  
  
    
	@Autowired


  		




  12
  
  
    
	private HttpConnector httpConnector;


  		




  13
  
  
    

  		




  14
  17
  
    
	@Override


  		




  15
  18
  
    
	protected BufferedInputStream getBufferedInputStream(final String baseUrl) throws CollectorServiceException {


  		




  
  19
  
    
		final HttpGet method = new HttpGet(baseUrl);


  		




  16
  20
  
    

  		




  17
  
  
    
		String res = httpConnector.getInputSource(baseUrl);


  		




  18
  
  
    
		return new BufferedInputStream(IOUtils.toInputStream(res));


  		




  
  21
  
    
		try(CloseableHttpResponse response = HttpClients.createDefault().execute(method)) {


  		




  
  22
  
    

  		




  
  23
  
    
			int responseCode = response.getStatusLine().getStatusCode();


  		




  
  24
  
    

  		




  
  25
  
    
			if (HttpStatus.SC_OK != responseCode) {


  		




  
  26
  
    
				throw new CollectorServiceException("Error " + responseCode + " dowloading url: " + baseUrl);


  		




  
  27
  
    
			}


  		




  
  28
  
    

  		




  
  29
  
    
			byte[] content = IOUtils.toByteArray(response.getEntity().getContent());


  		




  
  30
  
    

  		




  
  31
  
    
			try(InputStream in = new ByteArrayInputStream(content)) {


  		




  
  32
  
    
				return new BufferedInputStream(in);


  		




  
  33
  
    
			}


  		




  
  34
  
    
		} catch (IOException e) {


  		




  
  35
  
    
			throw new CollectorServiceException("Error dowloading url: " + baseUrl);


  		




  
  36
  
    
		}


  		




  19
  37
  
    
	}


  		




  20
  38
  
    

  		




  21
  39
  
    
}


  		












  

    
      modules/dnet-modular-collector-service/trunk/src/main/java/eu/dnetlib/data/collector/plugins/httplist/HttpListIterator.java
    
  





  1
  1
  
    
package eu.dnetlib.data.collector.plugins.httplist;


  		




  2
  2
  
    

  		




  
  3
  
    
import org.apache.commons.io.IOUtils;


  		




  
  4
  
    
import org.apache.commons.lang3.StringUtils;


  		




  
  5
  
    
import org.apache.http.HttpStatus;


  		




  
  6
  
    
import org.apache.http.client.methods.CloseableHttpResponse;


  		




  
  7
  
    
import org.apache.http.client.methods.HttpGet;


  		




  
  8
  
    
import org.apache.http.impl.client.CloseableHttpClient;


  		




  
  9
  
    
import org.apache.http.impl.client.HttpClients;


  		




  
  10
  
    

  		




  3
  11
  
    
import java.io.BufferedInputStream;


  		




  4
  12
  
    
import java.io.BufferedReader;


  		




  5
  13
  
    
import java.io.IOException;


  		




  6
  14
  
    
import java.io.StringReader;


  		




  7
  15
  
    
import java.util.Iterator;


  		




  8
  16
  
    

  		




  9
  
  
    
import eu.dnetlib.data.collector.plugins.HttpConnector;


  		




  10
  
  
    
import eu.dnetlib.data.collector.rmi.CollectorServiceException;


  		




  11
  
  
    
import org.apache.commons.io.IOUtils;


  		




  12
  
  
    
import org.apache.commons.lang3.StringUtils;


  		




  13
  
  
    

  		




  14
  17
  
    
public class HttpListIterator implements Iterator<String> {


  		




  15
  18
  
    

  		




  16
  
  
    
	private HttpConnector httpConnector;


  		




  
  19
  
    
	final CloseableHttpClient client = HttpClients.createDefault();


  		




  17
  20
  
    

  		




  18
  21
  
    
	private String baseUrl;


  		




  19
  22
  
    
	private String currentLine;


  		




  20
  23
  
    
	private BufferedReader reader;


  		




  21
  24
  
    

  		




  22
  
  
    
	public HttpListIterator(final String baseUrl, final String listAddress, final HttpConnector httpConnector) {


  		




  23
  
  
    
		this.httpConnector = httpConnector;


  		




  24
  
  
    
		this.baseUrl = baseUrl;


  		




  
  25
  
    
	public HttpListIterator(final String baseUrl, final String listAddress) {


  		




  25
  26
  
    
		try {


  		




  
  27
  
    
			this.baseUrl = baseUrl;


  		




  26
  28
  
    
			this.reader = new BufferedReader(new StringReader(download(listAddress)));


  		




  27
  29
  
    
			this.currentLine = reader.readLine();


  		




  28
  30
  
    
		} catch (Exception e) {


  		




  ......




  53
  55
  
    
	}


  		




  54
  56
  
    

  		




  55
  57
  
    
	private String download(final String url) {


  		




  56
  
  
    
		try {


  		




  57
  
  
    
			return IOUtils.toString(new BufferedInputStream(httpConnector.getInputSourceAsStream(url)));


  		




  58
  
  
    
		} catch(Exception e) {


  		




  59
  
  
    
			throw new RuntimeException(String.format("Error downloading url: %s, error: %s", url, e.getMessage()));


  		




  
  58
  
    
		try(CloseableHttpResponse response = client.execute(new HttpGet(url))) {


  		




  
  59
  
    

  		




  
  60
  
    
			final int responseCode = response.getStatusLine().getStatusCode();


  		




  
  61
  
    

  		




  
  62
  
    
			if (responseCode == HttpStatus.SC_OK) {


  		




  
  63
  
    
				return IOUtils.toString(new BufferedInputStream(response.getEntity().getContent()));


  		




  
  64
  
    
			} else {


  		




  
  65
  
    
				throw new RuntimeException("Error " + responseCode + " dowloading url: " + url);


  		




  
  66
  
    
			}


  		




  
  67
  
    

  		




  
  68
  
    
		} catch (IOException e) {


  		




  
  69
  
    
			throw new RuntimeException("Error dowloading url: " + url, e);


  		




  60
  70
  
    
		}


  		




  61
  71
  
    
	}


  		




  62
  72
  
    

  		




  63
  73
  
    
	@Override


  		




  64
  74
  
    
	public void remove() {}


  		




  65
  75
  
    

  		




  66
  
  
    
	public HttpConnector getHttpConnector() {


  		




  67
  
  
    
		return httpConnector;


  		




  68
  
  
    
	}


  		




  69
  
  
    

  		




  70
  
  
    
	public void setHttpConnector(final HttpConnector httpConnector) {


  		




  71
  
  
    
		this.httpConnector = httpConnector;


  		




  72
  
  
    
	}


  		




  73
  76
  
    
}


  		












  

    
      modules/dnet-modular-collector-service/trunk/pom.xml
    
  





  3
  3
  
    
	<parent>


  		




  4
  4
  
    
		<groupId>eu.dnetlib</groupId>


  		




  5
  5
  
    
		<artifactId>dnet45-parent</artifactId>


  		




  6
  
  
    
		<version>1.0.0-SNAPSHOT</version>


  		




  
  6
  
    
		<version>1.0.0</version>


  		




  7
  7
  
    
		<relativePath />


  		




  8
  8
  
    
	</parent>


  		




  9
  9
  
    
	<modelVersion>4.0.0</modelVersion>


  		




  ......




  50
  50
  
    
		<dependency>


  		




  51
  51
  
    
			<groupId>commons-net</groupId>


  		




  52
  52
  
    
			<artifactId>commons-net</artifactId>


  		




  53
  
  
    
			<version>[3.6, 4.0)</version>


  		




  
  53
  
    
			<version>3.3</version>


  		




  54
  54
  
    
		</dependency>


  		




  55
  55
  
    
		<dependency>


  		




  56
  56
  
    
			<groupId>org.apache.commons</groupId>


  		




  ......




  63
  63
  
    
			<version>${mockito.version}</version>


  		




  64
  64
  
    
			<scope>test</scope>


  		




  65
  65
  
    
		</dependency>


  		




  66
  
  
    
		<!--<dependency>-->


  		




  67
  
  
    
			<!--<groupId>commons-httpclient</groupId>-->


  		




  68
  
  
    
			<!--<artifactId>commons-httpclient</artifactId>-->


  		




  69
  
  
    
			<!--<version>3.1</version>-->


  		




  70
  
  
    
		<!--</dependency>-->


  		




  71
  66
  
    
		<dependency>


  		




  72
  67
  
    
			<groupId>org.apache.httpcomponents</groupId>


  		




  73
  68
  
    
			<artifactId>httpclient</artifactId>


  		




  74
  
  
    
			<version>[4.5.6,5.0.0)</version>


  		




  
  69
  
    
			<version>4.5.3</version>


  		




  75
  70
  
    
		</dependency>


  		




  76
  71
  
    
		<dependency>


  		




  77
  72
  
    
			<groupId>com.google.code.gson</groupId>


  		












Also available in:  Unified diff