D-Net

package eu.dnetlib.data.collector.plugins.oai;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

import eu.dnetlib.data.collector.plugins.oai.engine.HttpConnector;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

public class HttpConnectorTest {

	private HttpConnector connector;

	private static final String URL = "https://researchdata.ands.org.au/registry/services/oai?verb=Identify";

	@Before

	public void setUp() {

		connector = new HttpConnector();

		connector.initTrustManager();

		connector.setMaxNumberOfRetry(1);

	}

	@Test

	@Ignore

	public void testGetInputSource() throws CollectorServiceException {

		System.out.println(connector.getInputSource(URL));

	}

}

package eu.dnetlib.data.collector.plugins.oai.engine;

import java.io.IOException;

import java.io.InputStream;

import java.net.CookieHandler;

import java.net.CookieManager;

import java.net.CookiePolicy;

import java.net.HttpURLConnection;

import java.net.URL;

import java.security.GeneralSecurityException;

import java.security.KeyManagementException;

import java.security.NoSuchAlgorithmException;

import java.security.cert.CertificateException;

import java.security.cert.X509Certificate;

import java.util.List;

import java.util.Map;

import javax.net.ssl.HttpsURLConnection;

import javax.net.ssl.SSLContext;

import javax.net.ssl.TrustManager;

import javax.net.ssl.X509TrustManager;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang.math.NumberUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import eu.dnetlib.data.collector.plugin.CollectorPluginErrorLogList;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

/**

 * @author jochen, michele, andrea

 *

 */

public class HttpConnector {

	private static final Log log = LogFactory.getLog(HttpConnector.class);

	private int maxNumberOfRetry = 6;

	private int defaultDelay = 120; // seconds

	private int readTimeOut = 120; // seconds

	private String userAgent = "Mozilla/5.0 (compatible; OAI; +http://www.openaire.eu)";

    public HttpConnector(){

    	CookieHandler.setDefault(new CookieManager(null, CookiePolicy.ACCEPT_ALL)); 

    }

	/**

	 * @param requestUrl

	 * @return the content of the downloaded resource

	 * @throws CollectorServiceException

	 */

	public String getInputSource(final String requestUrl) throws CollectorServiceException {

		return attemptDownload(requestUrl, 1, new CollectorPluginErrorLogList());

	}

	private String attemptDownload(final String requestUrl, final int retryNumber, final CollectorPluginErrorLogList errorList)

			throws CollectorServiceException {

		if (retryNumber > maxNumberOfRetry) { throw new CollectorServiceException("Max number of retries exceeded. Cause: \n " + errorList); }

		log.debug("Downloading " + requestUrl + " - try: " + retryNumber);

		try {

			InputStream input = null;

			try {

				final HttpURLConnection urlConn = (HttpURLConnection) new URL(requestUrl).openConnection();

				urlConn.setInstanceFollowRedirects(false);

				urlConn.setReadTimeout(readTimeOut * 1000);

                                urlConn.addRequestProperty("User-Agent", userAgent);

				if (log.isDebugEnabled()) {

					logHeaderFields(urlConn);

				}

				int retryAfter = obtainRetryAfter(urlConn.getHeaderFields());

				if (retryAfter > 0) {

					log.warn("waiting and repeating request after " + retryAfter + " sec.");

					Thread.sleep(retryAfter * 1000);

					errorList.add("503 Service Unavailable");

					return attemptDownload(requestUrl, retryNumber + 1, errorList);

				} else if ((urlConn.getResponseCode() == HttpURLConnection.HTTP_MOVED_PERM) || (urlConn.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP)) {

					final String newUrl = obtainNewLocation(urlConn.getHeaderFields());

					log.info("The requested url has been moved to " + newUrl);

					errorList.add(String.format("%s %s. Moved to: %s", urlConn.getResponseCode(), urlConn.getResponseMessage(), newUrl));

					return attemptDownload(newUrl, retryNumber + 1, errorList);

				} else if (urlConn.getResponseCode() != HttpURLConnection.HTTP_OK) {

					log.error(String.format("HTTP error: %s %s", urlConn.getResponseCode(), urlConn.getResponseMessage()));

					Thread.sleep(defaultDelay * 1000);

					errorList.add(String.format("%s %s", urlConn.getResponseCode(), urlConn.getResponseMessage()));

					return attemptDownload(requestUrl, retryNumber + 1, errorList);

				} else {

					input = urlConn.getInputStream();

					return IOUtils.toString(input);

				}

			} catch (IOException e) {

				log.error("error while retrieving from http-connection occured: " + e, e);

				Thread.sleep(defaultDelay * 1000);

				errorList.add(e.getMessage());

				return attemptDownload(requestUrl, retryNumber + 1, errorList);

			} finally {

				IOUtils.closeQuietly(input);

			}

		} catch (InterruptedException e) {

			throw new CollectorServiceException(e);

		}

	}

	private void logHeaderFields(final HttpURLConnection urlConn) throws IOException {

		log.debug("StatusCode: " + urlConn.getResponseMessage());

		for (Map.Entry<String, List<String>> e : urlConn.getHeaderFields().entrySet()) {

			if (e.getKey() != null) {

				for (String v : e.getValue()) {

					log.debug("  key: " + e.getKey() + " - value: " + v);

				}

			}

		}

	}

	private int obtainRetryAfter(final Map<String, List<String>> headerMap) {

		for (String key : headerMap.keySet()) {

			if ((key != null) && key.toLowerCase().equals("retry-after") && (headerMap.get(key).size() > 0) && NumberUtils.isNumber(headerMap.get(key).get(0))) { return Integer

					.parseInt(headerMap.get(key).get(0)) + 10; }

		}

		return -1;

	}

	private String obtainNewLocation(final Map<String, List<String>> headerMap) throws CollectorServiceException {

		for (String key : headerMap.keySet()) {

			if ((key != null) && key.toLowerCase().equals("location") && (headerMap.get(key).size() > 0)) { return headerMap.get(key).get(0); }

		}

		throw new CollectorServiceException("The requested url has been MOVED, but 'location' param is MISSING");

	}

	/**

	 * register for https scheme; this is a workaround and not intended for the use in trusted environments

	 *

	 * @throws NoSuchAlgorithmException

	 * @throws KeyManagementException

	 */

	public void initTrustManager() {

		final X509TrustManager tm = new X509TrustManager() {

			@Override

			public void checkClientTrusted(final X509Certificate[] xcs, final String string) throws CertificateException {}

			@Override

			public void checkServerTrusted(final X509Certificate[] xcs, final String string) throws CertificateException {}

			@Override

			public X509Certificate[] getAcceptedIssuers() {

				return null;

			}

		};

		try {

			final SSLContext ctx = SSLContext.getInstance("TLS");

			ctx.init(null, new TrustManager[] { tm }, null);

			HttpsURLConnection.setDefaultSSLSocketFactory(ctx.getSocketFactory());

		} catch (GeneralSecurityException e) {

			log.fatal(e);

			throw new IllegalStateException(e);

		}

	}

	public int getMaxNumberOfRetry() {

		return maxNumberOfRetry;

	}

	public void setMaxNumberOfRetry(final int maxNumberOfRetry) {

		this.maxNumberOfRetry = maxNumberOfRetry;

	}

	public int getDefaultDelay() {

		return defaultDelay;

	}

	public void setDefaultDelay(final int defaultDelay) {

		this.defaultDelay = defaultDelay;

	}

	public int getReadTimeOut() {

		return readTimeOut;

	}

	public void setReadTimeOut(final int readTimeOut) {

		this.readTimeOut = readTimeOut;

	}

}

import eu.dnetlib.data.collector.plugins.oai.engine.HttpConnector;

		class="eu.dnetlib.data.collector.plugins.oai.engine.HttpConnector"

import eu.dnetlib.data.collector.plugins.oai.engine.HttpConnector;

import eu.dnetlib.data.collector.plugins.oai.engine.HttpConnector;

import eu.dnetlib.data.collector.plugins.oai.engine.HttpConnector;

import eu.dnetlib.data.collector.plugins.oai.engine.HttpConnector;

	<bean id="oaiHttpConnector" class="eu.dnetlib.data.collector.plugins.oai.engine.HttpConnector"

		init-method="initTrustManager"

		p:defaultDelay="${collector.oai.http.defaultDelay}"

		p:readTimeOut="${collector.oai.http.readTimeOut}"

		p:maxNumberOfRetry="${collector.oai.http.maxNumberOfRetry}" />

		p:httpConnector-ref="oaiHttpConnector" />

		p:httpConnector-ref="oaiHttpConnector" />

import eu.dnetlib.data.collector.plugins.oai.engine.HttpConnector;

import eu.dnetlib.data.collector.plugins.oai.engine.HttpConnector;

import eu.dnetlib.data.collector.plugins.oai.engine.HttpConnector;