D-Net

package eu.dnetlib.data.collector.plugins.httplist;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringUtils;

import org.apache.http.HttpStatus;

import org.apache.http.client.methods.CloseableHttpResponse;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.impl.client.CloseableHttpClient;

import org.apache.http.impl.client.HttpClients;

import java.io.BufferedInputStream;

import java.io.BufferedReader;

import java.io.IOException;

import java.io.StringReader;

import java.util.Iterator;

public class HttpListIterator implements Iterator<String> {

	final CloseableHttpClient client = HttpClients.createDefault();

	private String baseUrl;

	private String currentLine;

	private BufferedReader reader;

	public HttpListIterator(final String baseUrl, final String listAddress) {

		try {

			this.baseUrl = baseUrl;

			this.reader = new BufferedReader(new StringReader(download(listAddress)));

			this.currentLine = reader.readLine();

		} catch (Exception e) {

			throw new RuntimeException("Error creating iterator", e);

		}

	}

	@Override

	public synchronized boolean hasNext() {

		return StringUtils.isNotBlank(currentLine);

	}

	@Override

	public synchronized String next() {

		try {

			if (StringUtils.isNotBlank(currentLine)) {

				return download(baseUrl + currentLine);

			} else {

				throw new RuntimeException("Iterator has reached the end");

			}

		} finally {

			try {

				this.currentLine = reader.readLine();

			} catch (IOException e) {

				throw new RuntimeException("Error obtaining next element " + currentLine, e);

			}

		}

	}

	private String download(final String url) {

		try(CloseableHttpResponse response = client.execute(new HttpGet(url))) {

			final int responseCode = response.getStatusLine().getStatusCode();

			if (responseCode == HttpStatus.SC_OK) {

				return IOUtils.toString(new BufferedInputStream(response.getEntity().getContent()));

			} else {

				throw new RuntimeException("Error " + responseCode + " dowloading url: " + url);

			}

		} catch (IOException e) {

			throw new RuntimeException("Error dowloading url: " + url, e);

		}

	}

	@Override

	public void remove() {}

}