D-Net

package eu.dnetlib.data.collector.plugins.schemaorg.sitemapindex;

import eu.dnetlib.data.collector.plugins.schemaorg.Utils;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import java.net.URL;

import java.nio.charset.Charset;

import java.util.*;

public class SitemapIndexIterator implements Iterator<String> {

	private static final Log log = LogFactory.getLog(SitemapIndexIterator.class);

	public static class Options {

		private URL indexUrl;

		private Charset charset;

		public Options(){}

		public Options(URL indexUrl, Charset charset){

			this.indexUrl = indexUrl;

			this.charset = charset;

		}

		public URL getIndexUrl() {

			return indexUrl;

		}

		public void setIndexUrl(URL indexUrl) {

			this.indexUrl = indexUrl;

		}

		public Charset getCharset() {

			return charset;

		}

		public void setCharset(Charset charset) {

			this.charset = charset;

		}

	}

	private Options options;

	private Queue<String> sitemapFiles;

	public SitemapIndexIterator(Options options) {

		this.options = options;

	}

	public void bootstrap() {

		List<String> files = null;

		try {

			log.debug("bootstrapping sitemapindex index access");

			String sitemapIndexPayload = IOUtils.toString(this.options.getIndexUrl(), this.options.getCharset());

			log.debug(String.format("sitemapindex payload is: %s", sitemapIndexPayload));

			files = Utils.collectAsStrings(sitemapIndexPayload, "/sitemapindex/sitemap/loc/text()");

			log.debug(String.format("extracted %d sitemapindex files", files.size()));

		}catch(Exception ex){

			log.error("problem bootstrapping sitemapindex index access. returning 0 files", ex);

			files = new ArrayList<>();

		}

		this.sitemapFiles = new PriorityQueue<String>(files);

	}

	@Override

	public boolean hasNext() {

		return !this.sitemapFiles.isEmpty();

	}

	@Override

	public String next() {

		return this.sitemapFiles.poll();

	}

}