D-Net

/**

 *

 * log.debug(...) equal to  log.trace(...) in the application-logs

 * known bug: at resumptionType 'discover' if the (resultTotal % resultSizeValue) == 0 the collecting fails -> change the resultSizeValue

package eu.dnetlib.data.collector.plugins.rest;

import java.io.InputStream;

import java.util.Iterator;

import java.util.Queue;

import javax.xml.transform.OutputKeys;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerConfigurationException;

import javax.xml.transform.TransformerFactory;

import javax.xml.transform.stream.StreamResult;

import javax.xml.xpath.XPathConstants;

import javax.xml.xpath.XPathExpression;

import javax.xml.xpath.XPathExpressionException;

import javax.xml.xpath.XPathFactory;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.LogFactory;

 */

public class RestIterator implements Iterator<String> {

	private String baseUrl;

	private String resumptionType;

	private String resumptionParam;

	private String resultFormatValue;

	private String queryParams;

	private int resultTotal = -1;

	private String resumptionStr = Integer.toString(resumptionInt);  // string resumption token (first record to harvest or token scanned from results)

	private InputStream resultStream;

	private Transformer transformer;

	private XPath xpath;

	private XPathExpression xprResumptionPath;

	private String querySize;

			final String baseUrl,

			final String resumptionType,

			final String resumptionParam,

			final String resumptionXpath,

			final String resultTotalXpath,

			final String resultFormatParam,

			final String resultFormatValue,

			final String resultSizeParam,

			final String entityXpath

		this.baseUrl = baseUrl;

		this.resumptionType = resumptionType;

		this.resumptionParam = resumptionParam;

		this.resultFormatValue = resultFormatValue;

		this.queryParams = queryParams;

		try {

			throw new IllegalStateException("xml transformation init failed: " + e.getMessage());

        initQueue();

	}

                transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount","3");

		xpath              = XPathFactory.newInstance().newXPath();

		xprEntity          = xpath.compile(entityXpath);

	private void initQueue() {

                //  change at 2018-08-30 and remove resumptionParam&-Type from "query" URL because first request starts mostly at the beginning.

		query = baseUrl + "?" + queryParams + querySize + queryFormat;

	private void disconnect() {

		// TODO close inputstream

	}

	/* (non-Javadoc)

	 * @see java.util.Iterator#hasNext()

	 */

	@Override

	public boolean hasNext() {

			return false;

		} else {

			return true;

		}

	}

	/* (non-Javadoc)

	 * @see java.util.Iterator#next()

	 */

	@Override

	public String next() {

			while (recordQueue.isEmpty() && !query.isEmpty() ) {

				try {

				}

			}

			return recordQueue.poll();

		}

	}

	/*

	 * download page and return nextQuery

	 */

	private String downloadPage(String query) throws CollectorServiceException{

                int urlOldResumptionSize = 0;

                        resultStream = qUrl.openStream();

				log.trace("before inputStream: " + resultXml);

                                log.trace("after cleaning: " + resultXml);

			}

                            resultNode = (Node) xpath.evaluate("/", new InputSource(resultStream), XPathConstants.NODE);

                            nodeList   = (NodeList) xprEntity.evaluate(resultNode, XPathConstants.NODESET);

                                StringWriter sw = new StringWriter();

                                transformer.transform(new DOMSource(nodeList.item(i)), new StreamResult(sw));

                                recordQueue.add(sw.toString());

                            }

                        if("scan".equals(resumptionType.toLowerCase())) { resumptionStr = xprResumptionPath.evaluate(resultNode);}

			if("count".equals(resumptionType.toLowerCase())){ resumptionStr = Integer.toString(resumptionInt); }

*/

                                            break;

                                            break;

                                            String[] arrayQUrlArgument = qUrlArgument.split("&");

                                            for(String arrayUrlArgStr : arrayQUrlArgument ) {

                                                if(arrayUrlArgStr.startsWith(resumptionParam)) {

                                                    String[] resumptionKeyValue = arrayUrlArgStr.split("=");

                                                    urlOldResumptionSize = Integer.parseInt(resumptionKeyValue[1]);

                                                    log.debug("discover OldResumptionSize from Url: " + urlOldResumptionSize);

                                                }

                                            }

                                            //

                                            if(   ( (emptyXml.toLowerCase()).equals(resultXml.toLowerCase()) )

                                               || ( (nodeList != null) && (nodeList.getLength() < resultSizeValue) )

                                              ) {

                                                resumptionStr = Integer.toString(resumptionInt);

                                                resultTotal   = resumptionInt+1;

                            case "page":

                                            pagination += 1;

                                            if (nodeList != null) {

                                                discoverResultSize += nodeList.getLength();

                                            } else {

                                                resultTotal = discoverResultSize;

                                                pagination  = discoverResultSize;

                                            }

                                            resumptionInt = pagination;

                                            resumptionStr = Integer.toString(resumptionInt);

				resultTotal = Integer.parseInt(xprResultTotalPath.evaluate(resultNode));

			log.info("resInt: " + resumptionInt);

			log.error(e);

			throw new IllegalStateException("collection failed: " + e.getMessage());

	}

        /**

         * check W3C XML syntax: https://www.w3.org/TR/2006/REC-xml11-20060816/#sec-starttags for valid tag names

         * and work-around for the JSON to XML converting of org.json.XML-package.

         *

        private String syntaxConvertJsonKeyNamens(String jsonInput) {

            while (jsonInput.matches(".*\"([^\"]*)\\s+([^\"]*)\":.*")) {

                jsonInput = jsonInput.replaceAll("\"([^\"]*)\\s+([^\"]*)\":", "\"$1_$2\":");

            }

            // replace forward-slash (sign '/' ) in JSON Names with '_'

            while (jsonInput.matches(".*\"([^\"]*)/([^\"]*)\":.*")) {

                jsonInput = jsonInput.replaceAll("\"([^\"]*)/([^\"]*)\":", "\"$1_$2\":");

            }

            // replace '(' in JSON Names with ''

            while (jsonInput.matches(".*\"([^\"]*)[(]([^\"]*)\":.*")) {

                jsonInput = jsonInput.replaceAll("\"([^\"]*)[(]([^\"]*)\":", "\"$1$2\":");

            }

            while (jsonInput.matches(".*\"([^\"]*)[)]([^\"]*)\":.*")) {

                jsonInput = jsonInput.replaceAll("\"([^\"]*)[)]([^\"]*)\":", "\"$1$2\":");

            }

            while (jsonInput.matches(".*\"([^\"][0-9])([^\"]*)\":.*")) {

                jsonInput = jsonInput.replaceAll("\"([^\"][0-9])([^\"]*)\":", "\"n_$1$2\":");

            }

            // replace ':' between number like '2018-08-28T11:05:00Z' in JSON keynames with ''

            while (jsonInput.matches(".*\"([^\"]*[0-9]):([0-9][^\"]*)\":.*")) {

                jsonInput = jsonInput.replaceAll("\"([^\"]*[0-9]):([0-9][^\"]*)\":", "\"$1$2\":");

            }

//            while (jsonInput.matches(".*\"([^\"]*),([^\"]*)\":.*")) {

//                jsonInput = jsonInput.replaceAll("\"([^\"]*),([^\"]*)\":", "\"$1.$2\":");

//            }

            // replace '=' in JSON Keynames with '-'

            while (jsonInput.matches(".*\"([^\"]*)=([^\"]*)\":.*")) {

                jsonInput = jsonInput.replaceAll("\"([^\"]*)=([^\"]*)\":", "\"$1-$2\":");

            }

        }

        /**

         *

         */

            while ( bufferStr.matches(".*<([^<>].*),(.)>.*") ) {

                bufferStr = bufferStr.replaceAll("<([^<>.*),(.*)>", "<$1$2>");

            }

//                bufferStr = bufferStr.replaceAll("&#x1([0-9a-fA-F])", "");

//            }

            return bufferStr;

        }