D-Net

package eu.dnetlib.data.collector.plugins.fairsharing;

import java.util.Iterator;

import java.util.Queue;

import java.util.concurrent.PriorityBlockingQueue;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.http.client.methods.CloseableHttpResponse;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.impl.client.CloseableHttpClient;

import org.apache.http.impl.client.HttpClients;

import org.json.JSONObject;

import org.json.XML;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

public class FairSharingIterator implements Iterator<String> {

	private static final Log log = LogFactory.getLog(FairSharingIterator.class); // NOPMD by marko on 11/24/08 5:02 PM

	private final Queue<String> queue = new PriorityBlockingQueue<>();

	private final String baseUrl;

	private final String authCode;

	private final int pageSize;

	private String nextUrl;

	private boolean started;

	public FairSharingIterator(final String baseUrl, final String authCode, final int pageSize) {

		this.baseUrl = baseUrl;

		this.authCode = authCode;

		this.pageSize = pageSize;

		this.started = false;

	}

	private void verifyStarted() {

		if (!this.started) {

			this.started = true;

			try {

				final String url = baseUrl + "/fairsharing_records/?page%5Bnumber%5D=1&page%5Bsize%5D=" + pageSize;

				this.nextUrl = downloadPage(url);

			} catch (final CollectorServiceException e) {

				throw new RuntimeException(e);

			}

		}

	}

	@Override

	public boolean hasNext() {

		synchronized (queue) {

			verifyStarted();

			return !queue.isEmpty();

		}

	}

	@Override

	public String next() {

		synchronized (queue) {

			verifyStarted();

			final String res = queue.poll();

			while (queue.isEmpty() && nextUrl != null && !nextUrl.isEmpty()) {

				try {

					nextUrl = downloadPage(nextUrl);

				} catch (final CollectorServiceException e) {

					throw new RuntimeException(e);

				}

			}

			return res;

		}

	}

	@Override

	public void remove() {}

	private String downloadPage(final String url) throws CollectorServiceException {

		log.debug("Fetching url: " + url);

		final HttpGet req = new HttpGet(url);

		req.addHeader("Accept", "application/json");

		req.addHeader("Content-Type", "application/json");

		req.addHeader("Authorization", "Bearer " + authCode);

		try (final CloseableHttpClient client = HttpClients.createDefault()) {

			try (final CloseableHttpResponse response = client.execute(req)) {

				final String content = IOUtils.toString(response.getEntity().getContent());

				final JSONObject obj = new JSONObject(content);

				obj.getJSONArray("data")

					.forEach(x -> queue.add(XML.toString(x, "record")));

				final JSONObject links = obj.getJSONObject("links");

				return links.isNull("next") ? null : links.getString("next");

			}

		} catch (final Exception e) {

			throw new CollectorServiceException("Error perfoming call fro login", e);

		}

	}

}