D-Net

package eu.dnetlib.data.collector.plugins.httplist;

import java.io.BufferedInputStream;

import java.io.BufferedReader;

import java.io.IOException;

import java.io.StringReader;

import java.util.Iterator;

import eu.dnetlib.rmi.data.CollectorServiceRuntimeException;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringUtils;

import org.apache.http.HttpStatus;

import org.apache.http.client.methods.CloseableHttpResponse;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.impl.client.HttpClients;

import org.apache.http.impl.client.CloseableHttpClient;

public class HttpListIterator implements Iterator<String> {

	private final CloseableHttpClient client = HttpClients.createDefault();

	private String baseUrl;

	private String currentLine;

	private BufferedReader reader;

	public HttpListIterator(final String baseUrl, final String listAddress) {

		try {

			this.baseUrl = baseUrl;

			this.reader = new BufferedReader(new StringReader(download(listAddress)));

			this.currentLine = reader.readLine();

		} catch (Exception e) {

			throw new CollectorServiceRuntimeException("Error creating iterator", e);

		}

	}

	@Override

	public synchronized boolean hasNext() {

		return StringUtils.isNotBlank(currentLine);

	}

	@Override

	public synchronized String next() {

		try {

			if (StringUtils.isNotBlank(currentLine)) {

				return download(baseUrl + currentLine);

			} else {

				throw new CollectorServiceRuntimeException("Iterator has reached the end");

			}

		} finally {

			try {

				this.currentLine = reader.readLine();

			} catch (IOException e) {

				throw new CollectorServiceRuntimeException("Error obtaining next element " + currentLine, e);

			}

		}

	}

	private String download(final String url) {

		final HttpGet method = new HttpGet(url);

		try(CloseableHttpResponse response = client.execute(method)) {

			int statusCode = response.getStatusLine().getStatusCode();

			if (HttpStatus.SC_OK == statusCode) {

				return IOUtils.toString(new BufferedInputStream(response.getEntity().getContent()));

			} else {

				throw new CollectorServiceRuntimeException("Error " + statusCode + " dowloading url: " + url);

			}

		} catch (IOException e) {

			throw new CollectorServiceRuntimeException("Error downloading url: " + url);

		}

	}

	@Override

	public void remove() {

		throw new UnsupportedOperationException();

	}

}