D-Net

package eu.dnetlib.data.collector.plugins.projects.gtr2;

import java.util.Iterator;

import java.util.NoSuchElementException;

import java.util.concurrent.ArrayBlockingQueue;

import java.util.concurrent.ExecutorService;

import java.util.concurrent.Executors;

import java.util.concurrent.TimeUnit;

import com.ximpleware.AutoPilot;

import com.ximpleware.VTDGen;

import com.ximpleware.VTDNav;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.CollectorServiceRuntimeException;

import eu.dnetlib.enabling.resultset.SizedIterable;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.joda.time.DateTime;

import org.joda.time.format.DateTimeFormat;

import org.joda.time.format.DateTimeFormatter;

import eu.dnetlib.data.collector.plugins.HttpConnector;

/**

 * Created by alessia on 28/11/16.

 */

public class Gtr2ProjectsIterable implements SizedIterable<String> {

	public static final String TERMINATOR = "ARNOLD";

	public static final int WAIT_END_SECONDS = 120;

	public static final int PAGE_SZIE = 20;

	private static final Log log = LogFactory.getLog(Gtr2ProjectsIterable.class);

	private String queryURL;

	private int total = -1;

	private int startFromPage = 1;

	private int endAtPage;

	private VTDGen vg;

	private VTDNav vn;

	private AutoPilot ap;

	private String namespaces;

	private boolean incremental = false;

	private DateTime fromDate;

	private DateTimeFormatter simpleDateTimeFormatter = DateTimeFormat.forPattern("yyyy-MM-dd");

	private ArrayBlockingQueue<String> projects = new ArrayBlockingQueue<String>(20);

	//private boolean finished = false;

	private final ExecutorService es = Executors.newFixedThreadPool(PAGE_SZIE);

	private String nextElement = null;

	private HttpConnector connector;

	public Gtr2ProjectsIterable(final String baseUrl, final String fromDate) throws CollectorServiceException {

		prepare(baseUrl, fromDate);

		fillInfo(true);

	}

	public Gtr2ProjectsIterable(final String baseUrl, final String fromDate, final int startFromPage, final int endAtPage) throws CollectorServiceException {

		prepare(baseUrl, fromDate);

		this.setStartFromPage(startFromPage);

		this.setEndAtPage(endAtPage);

		fillInfo(false);

	}

	private void prepare(final String baseUrl, final String fromDate) {

		connector = new HttpConnector();

		queryURL = baseUrl + "/projects";

		vg = new VTDGen();

		this.incremental = StringUtils.isNotBlank(fromDate);

		if (incremental) {

			// I expect fromDate in the format 'yyyy-MM-dd'. See class eu.dnetlib.msro.workflows.nodes.collect.FindDateRangeForIncrementalHarvestingJobNode

			this.fromDate = DateTime.parse(fromDate, simpleDateTimeFormatter);

			log.debug("fromDate string: " + fromDate + " -- parsed: " + this.fromDate.toString());

		}

	}

	@Override

	public int getNumberOfElements() {

		return total;

	}

	private void fillInfo(final boolean all) throws CollectorServiceException {

		try {

			// log.debug("Getting hit count from: " + queryURL);

			byte[] bytes = connector.getInputSource(queryURL).getBytes("UTF-8");

			vg.setDoc(bytes);

			vg.parse(false);

			//vg.parseHttpUrl(queryURL, false);

			initParser();

			String hitCount = vn.toNormalizedString(vn.getAttrVal("totalSize"));

			String totalPages = vn.toNormalizedString(vn.getAttrVal("totalPages"));

			namespaces = "xmlns:ns1=\"" + vn.toNormalizedString(vn.getAttrVal("ns1")) + "\" ";

			namespaces += "xmlns:ns2=\"" + vn.toNormalizedString(vn.getAttrVal("ns2")) + "\" ";

			namespaces += "xmlns:ns3=\"" + vn.toNormalizedString(vn.getAttrVal("ns3")) + "\" ";

			namespaces += "xmlns:ns4=\"" + vn.toNormalizedString(vn.getAttrVal("ns4")) + "\" ";

			namespaces += "xmlns:ns5=\"" + vn.toNormalizedString(vn.getAttrVal("ns5")) + "\" ";

			namespaces += "xmlns:ns6=\"" + vn.toNormalizedString(vn.getAttrVal("ns6")) + "\" ";

			if (all) {

				setEndAtPage(Integer.parseInt(totalPages));

				total = Integer.parseInt(hitCount);

			}

			Thread ft = new Thread(new FillProjectList());

			ft.start();

			log.debug("Expected number of pages: " + (endAtPage - startFromPage + 1));

		} catch (NumberFormatException e) {

			log.error("Cannot set the total count or the number of pages");

			throw new CollectorServiceException(e);

		} catch (Throwable e) {

			throw new CollectorServiceException(e);

		}

	}

	@Override

	public Iterator<String> iterator() {

		return new Iterator<String>() {

			// The following is for debug only

			private int nextCounter = 0;

			@Override

			public boolean hasNext() {

				try {

					log.debug("hasNext?");

					if (nextElement == null) {

						nextElement = projects.poll(WAIT_END_SECONDS, TimeUnit.SECONDS);

						log.debug("Exit poll :-)");

					}

					return nextElement != null && !nextElement.equals(TERMINATOR);

				} catch (InterruptedException e) {

					throw new CollectorServiceRuntimeException(e);

				}

			}

			@Override

			public String next() {

				nextCounter++;

				log.debug(String.format("Calling next %s times.", nextCounter));

				if (nextElement == null) throw new NoSuchElementException();

				else {

					String res = nextElement;

					nextElement = null;

					return res;

				}

			}

			@Override

			public void remove() {

				throw new UnsupportedOperationException();

			}

		};

	}

	private void initParser() {

		vn = vg.getNav();

		ap = new AutoPilot(vn);

	}

	public String getQueryURL() {

		return queryURL;

	}

	public void setQueryURL(final String queryURL) {

		this.queryURL = queryURL;

	}

	public int getTotal() {

		return total;

	}

	public void setTotal(final int total) {

		this.total = total;

	}

	public int getEndAtPage() {

		return endAtPage;

	}

	public void setEndAtPage(final int endAtPage) {

		this.endAtPage = endAtPage;

		log.debug("Overriding endAtPage to " + endAtPage);

	}

	public VTDGen getVg() {

		return vg;

	}

	public void setVg(final VTDGen vg) {

		this.vg = vg;

	}

	public VTDNav getVn() {

		return vn;

	}

	public void setVn(final VTDNav vn) {

		this.vn = vn;

	}

	public AutoPilot getAp() {

		return ap;

	}

	public void setAp(final AutoPilot ap) {

		this.ap = ap;

	}

	public String getNamespaces() {

		return namespaces;

	}

	public void setNamespaces(final String namespaces) {

		this.namespaces = namespaces;

	}

	public int getStartFromPage() {

		return startFromPage;

	}

	public void setStartFromPage(final int startFromPage) {

		this.startFromPage = startFromPage;

		log.debug("Overriding startFromPage to " + startFromPage);

	}

	private class FillProjectList implements Runnable {

		private boolean morePages = true;

		private int pageNumber = startFromPage;

		@Override

		public void run() {

			String resultPageUrl = "";

			try {

				do {

					resultPageUrl = getNextPageUrl();

					log.debug("Page: " + resultPageUrl);

					// clear VGen before processing the next file

					vg.clear();

					byte[] bytes = connector.getInputSource(resultPageUrl).getBytes("UTF-8");

					vg.setDoc(bytes);

					vg.parse(false);

					//vg.parseHttpUrl(resultPageUrl, false);

					initParser();

					ap.selectXPath("//project");

					int res;

					while ((res = ap.evalXPath()) != -1) {

						final String projectHref = vn.toNormalizedString(vn.getAttrVal("href"));

						Thread t = new Thread(new ParseProject(projectHref));

						t.setName("Thread for " + res);

						es.execute(t);

					}

					ap.resetXPath();

				} while (morePages);

				es.shutdown();

				es.awaitTermination(WAIT_END_SECONDS, TimeUnit.SECONDS);

				projects.put(TERMINATOR);

			} catch (Throwable e) {

				log.error("Exception processing " + resultPageUrl + "\n" + e.getMessage());

			}

		}

		private String getNextPageUrl() {

			String url = queryURL + "?p=" + pageNumber;

			if (pageNumber == endAtPage) {

				morePages = false;

			}

			pageNumber++;

			return url;

		}

	}

	private class ParseProject implements Runnable {

		VTDNav vn1;

		VTDGen vg1;

		private String projectRef;

		public ParseProject(String projectHref) {

			if(projectHref.contains("gtr.gtr")){

				projectHref = projectHref.replace("gtr.gtr","gtr");

			}

			projectRef = projectHref;

			vg1 = new VTDGen();

			try {

				byte[] bytes = connector.getInputSource(projectRef).getBytes("UTF-8");

				vg1.setDoc(bytes);

				vg1.parse(false);

				//vg1.parseHttpUrl(projectRef, false);

				vn1 = vg1.getNav();

			}catch(Throwable e){

				log.error("Exception processing " + projectRef + "\n" + e.getMessage());

			}

		}

		private int projectsUpdate(String attr) throws CollectorServiceException {

			try {

				int index = vn1.getAttrVal(attr);

				if (index != -1) {

					String d = vn1.toNormalizedString(index);

					DateTime recordDate = DateTime.parse(d.substring(0, d.indexOf("T")), simpleDateTimeFormatter);

					// updated or created after the last time it was collected

					if (recordDate.isAfter(fromDate)) {

						log.debug("New project to collect");

						return index;

					}

					return -1;

				}

				return index;

			} catch (Throwable e) {

				throw new CollectorServiceException(e);

			}

		}

		private String collectProject() throws CollectorServiceException {

			try {

				int p = vn1.getAttrVal("href");

				final String projectHref = vn1.toNormalizedString(p);

				log.debug("collecting project at " + projectHref);

				Gtr2Helper gtr2Helper = new Gtr2Helper();

				String projectPackage = gtr2Helper.processProject(vn1, namespaces);

				return projectPackage;

			} catch (Throwable e) {

				throw new CollectorServiceException(e);

			}

		}

		private boolean add(String attr) throws CollectorServiceException {

			return projectsUpdate(attr) != -1;

		}

		@Override

		public void run() {

			log.debug("Getting project info from " + projectRef);

			try {

				if (!incremental || (incremental && (add("created") || add("updated")))) {

					projects.put(collectProject());

					log.debug("Project enqueued " + projectRef);

				}

			} catch (Throwable e) {

				log.error("Error on ParseProject " + e.getMessage());

				throw new CollectorServiceRuntimeException(e);

			}

		}

	}

}