D-Net

/**

 *

 * log.debug(...) equal to  log.trace(...) in the application-logs

package eu.dnetlib.data.collector.plugins.rest;

import java.io.InputStream;

import java.util.Iterator;

import java.util.Queue;

import javax.xml.transform.OutputKeys;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerConfigurationException;

import javax.xml.transform.TransformerFactory;

import javax.xml.transform.stream.StreamResult;

import javax.xml.xpath.XPathConstants;

import javax.xml.xpath.XPathExpression;

import javax.xml.xpath.XPathExpressionException;

import javax.xml.xpath.XPathFactory;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.LogFactory;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

 * @date 2018-08-06

 */

public class RestIterator implements Iterator<String> {

	private String baseUrl;

	private String resumptionType;

	private String resumptionParam;

	private String resultFormatValue;

	private String queryParams;

	private int resultTotal = -1;

	private String resumptionStr = Integer.toString(resumptionInt);  // string resumption token (first record to harvest or token scanned from results)

	private InputStream resultStream;

	private Transformer transformer;

	private XPath xpath;

	private XPathExpression xprResumptionPath;

	private String querySize;

			final String baseUrl,

			final String resumptionType,

			final String resumptionParam,

			final String resumptionXpath,

			final String resultTotalXpath,

			final String resultFormatParam,

			final String resultFormatValue,

			final String resultSizeParam,

			final String entityXpath

		this.baseUrl = baseUrl;

		this.resumptionType = resumptionType;

		this.resumptionParam = resumptionParam;

		this.resultFormatValue = resultFormatValue;

		this.queryParams = queryParams;

		try {

			throw new IllegalStateException("xml transformation init failed: " + e.getMessage());

        initQueue();

	}

                transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount","3");

		xpath              = XPathFactory.newInstance().newXPath();

		xprEntity          = xpath.compile(entityXpath);

	private void initQueue() {

	private void disconnect() {

		// TODO close inputstream

	}

	/* (non-Javadoc)

	 * @see java.util.Iterator#hasNext()

	 */

	@Override

	public boolean hasNext() {

			return false;

		} else {

			return true;

		}

	}

	/* (non-Javadoc)

	 * @see java.util.Iterator#next()

	 */

	@Override

	public String next() {

			while (recordQueue.isEmpty() && !query.isEmpty() ) {

				try {

				}

			}

			return recordQueue.poll();

		}

	}

	/*

	 * download page and return nextQuery

	 */

	private String downloadPage(String query) throws CollectorServiceException{

		String resultXml = "";

                Node resultNode = null;

                int urlOldResumptionSize = 0;

                        resultStream = qUrl.openStream();

				//TODO move regex definitions as constant fields

//                                while(resultJson.matches(".*\"([^\"]*)\\s+([^\"]*)\":.*")){

//					resultJson = resultJson.replaceAll("\"([^\"]*)\\s+([^\"]*)\":", "\"$1_$2\":");

//				}

				resultXml = org.json.XML.toString(jsonObject,wrapName); // wrap xml in single root element

			}

                            resultNode = (Node) xpath.evaluate("/", new InputSource(resultStream), XPathConstants.NODE);

                            nodeList   = (NodeList) xprEntity.evaluate(resultNode, XPathConstants.NODESET);

                                StringWriter sw = new StringWriter();

                                transformer.transform(new DOMSource(nodeList.item(i)), new StreamResult(sw));

                                recordQueue.add(sw.toString());

                            }

                        if("scan".equals(resumptionType.toLowerCase())) { resumptionStr = xprResumptionPath.evaluate(resultNode);}

			if("count".equals(resumptionType.toLowerCase())){ resumptionStr = Integer.toString(resumptionInt); }

*/

                                            break;

                            case "count":   // begin at one step for all records

                                            break;

                            case "discover":   // length of results unknown

                                            String[] arrayQUrlArgument = qUrlArgument.split("&");

                                            for(String arrayUrlArgStr : arrayQUrlArgument ) {

                                                if(arrayUrlArgStr.startsWith(resumptionParam)) {

                                                    String[] resumptionKeyValue = arrayUrlArgStr.split("=");

                                                    urlOldResumptionSize = Integer.parseInt(resumptionKeyValue[1]);

                                                    log.debug("discover OldResumptionSize from Url: " + urlOldResumptionSize);

                                                }

                                            }

                                            //

                                            if(   ( (emptyXml.toLowerCase()).equals(resultXml.toLowerCase()) )

                                               || ( (nodeList != null) && (nodeList.getLength() < resultSizeValue) )

                                              ) {

                                                resumptionStr = Integer.toString(resumptionInt);

                                                resultTotal   = resumptionInt+1;

                            default:        // otherwise: abort

				resultTotal = Integer.parseInt(xprResultTotalPath.evaluate(resultNode));

			log.info("resInt: " + resumptionInt);

			log.error(e);

			throw new IllegalStateException("collection failed: " + e.getMessage());

	}

        /**

         * convert in Json-KeyName 'whitespace(s)' to '_' and '/' to '_', '(' and ')' to ''

         *

         * @param jsonInput

         * @return

         */

        private String syntaxConvertJsonKeyNamens(String jsonInput) {

            while (jsonInput.matches(".*\"([^\"]*)\\s+([^\"]*)\":.*")) {

                jsonInput = jsonInput.replaceAll("\"([^\"]*)\\s+([^\"]*)\":", "\"$1_$2\":");

            }

            // replace forward-slash (sign '/' ) in JSON Names with '_'

            while (jsonInput.matches(".*\"([^\"]*)/([^\"]*)\":.*")) {

                jsonInput = jsonInput.replaceAll("\"([^\"]*)/([^\"]*)\":", "\"$1_$2\":");

            }

            // replace '(' in JSON Names with ''

            while (jsonInput.matches(".*\"([^\"]*)[(]([^\"]*)\":.*")) {

                jsonInput = jsonInput.replaceAll("\"([^\"]*)[(]([^\"]*)\":", "\"$1$2\":");

            }

            // replace ')' in JSON Names with ''

            while (jsonInput.matches(".*\"([^\"]*)[)]([^\"]*)\":.*")) {

                jsonInput = jsonInput.replaceAll("\"([^\"]*)[)]([^\"]*)\":", "\"$1$2\":");

            }

        }