D-Net

/**

 * 

 */

package eu.dnetlib.data.collector.plugins.rest;

import java.io.InputStream;

import java.net.URL;

import java.util.Iterator;

import java.util.LinkedList;

import java.util.Queue;

import javax.xml.transform.OutputKeys;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerConfigurationException;

import javax.xml.transform.TransformerFactory;

import javax.xml.xpath.XPath;

import javax.xml.xpath.XPathConstants;

import javax.xml.xpath.XPathExpression;

import javax.xml.xpath.XPathExpressionException;

import javax.xml.xpath.XPathFactory;

import org.apache.commons.io.IOUtils;

import org.w3c.dom.Node;

import org.xml.sax.InputSource;

/**

 * @author Jochen Schirrwagen, Aenne Loehden

 *

 */

public class RestIterator implements Iterator<String> {

	private static final String wrapName = "recordWrap";

	private String baseUrl;

	private String resumptionType;

	private String resumptionParam;

	private String resultFormatValue;

	private String queryParams;

	private int resultSizeValue = 100;

	private Queue<String> queue;

	private int resumptionInt = 0;			// integer resumption token (first record to harvest)

	private int resultTotal = -1;

	private String resumptionStr = Integer.toString(resumptionInt);  // string resumption token (first record to harvest or token scanned from results)

	private InputStream resultStream;

	private Transformer transformer;

	private XPath xpath;

	private XPathExpression xprResultTotalPath;

	private XPathExpression xprResumptionPath;

	private String queryFormat;

	private String querySize;

	/*

	 * 

	 */

	public RestIterator(

			final String baseUrl,

			final String resumptionType,

			final String resumptionParam,

			final String resumptionXpath,

			final String resultTotalXpath,

			final String resultFormatParam,

			final String resultFormatValue,

			final String resultSizeParam,

			final String queryParams

			) {

		this.baseUrl = baseUrl;

		this.resumptionType = resumptionType;

		this.resumptionParam = resumptionParam;

		this.resultFormatValue = resultFormatValue;

		this.queryParams = queryParams;

        queryFormat = (resultFormatParam!="")? "&" + resultFormatParam + "=" + resultFormatValue : "";

        querySize = (resultSizeParam!="")? "&" + resultSizeParam + "=" + resultSizeValue : "";

		try {

			initXmlTransformation(resultTotalXpath, resumptionXpath);

		}catch(Exception exp) {

			throw new IllegalStateException("xml transformation init failed: " + exp.getMessage());

		}

        initQueue();

        updateQueue();

	}

	private void initXmlTransformation(String resultTotalXpath, String resumptionXpath) throws TransformerConfigurationException, XPathExpressionException{

		String resumpXpath = (resumptionXpath=="") ? "/" : resumptionXpath;

		transformer = TransformerFactory.newInstance().newTransformer();

        transformer.setOutputProperty(OutputKeys.INDENT,"yes"); 

        transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount","3");

		xpath = XPathFactory.newInstance().newXPath();

		xprResultTotalPath = xpath.compile(resultTotalXpath);

		xprResumptionPath = xpath.compile(resumpXpath);

	}

	private void initQueue() {

		queue = new LinkedList<String>();

	}

	private void disconnect() {

		// TODO close inputstream

	}

	private void updateQueue() {

        String query = baseUrl + "?" + queryParams + querySize + "&" + resumptionParam + "=" + resumptionStr + queryFormat;

        System.out.println("query: " + query);

        queue.add(query);

	}

	/* (non-Javadoc)

	 * @see java.util.Iterator#hasNext()

	 */

	@Override

	public boolean hasNext() {

		if (queue.isEmpty()) {

			disconnect();

			return false;

		} else {

			return true;

		}

	}

	/* (non-Javadoc)

	 * @see java.util.Iterator#next()

	 */

	@Override

	public String next() {

		// TODO Auto-generated method stub

		String nextQuery = queue.remove();

		String resultJson;

		String resultXml = "";

		try {

            resultStream = new URL(nextQuery).openStream();

			if(resultFormatValue == "json"){				

				resultJson = IOUtils.toString(resultStream,"UTF-8");

				// pre-clean json - rid spaces of element names (misinterpreted as elements with attributes in xml)

				while(resultJson.matches(".*\"([^\"]*)\\s+([^\"]*)\":.*")){

					resultJson = resultJson.replaceAll("\"([^\"]*)\\s+([^\"]*)\":", "\"$1_$2\":");

				}

				org.json.JSONObject jsonObject = new org.json.JSONObject(resultJson);

				resultXml = org.json.XML.toString(jsonObject,wrapName); // wrap xml in single root element

//				System.out.println(resultXml);

				resultStream = IOUtils.toInputStream(resultXml,"UTF-8");

			}

			InputSource inSource = new InputSource(resultStream);

			Node resultNode = (Node) xpath.evaluate("/", inSource, XPathConstants.NODE);

			resumptionInt += resultSizeValue;

			if(resumptionType=="scan"){ resumptionStr = xprResumptionPath.evaluate(resultNode);}

			if(resumptionType=="count"){ resumptionStr = Integer.toString(resumptionInt); }

			if (resultTotal == -1) {

				resultTotal = Integer.parseInt(xprResultTotalPath.evaluate(resultNode));

				System.out.println("resultTotal: " + resultTotal);

			}

			System.out.println("resultTotal: " + resultTotal);

			System.out.println("resInt: " + resumptionInt);

			if (resumptionInt < resultTotal) {

				updateQueue();

			}

			return resultXml;

		}catch(Exception exc) {

			exc.printStackTrace(System.err);

			throw new IllegalStateException("collection failed: " + exc.getMessage());

		}

	}

}