D-Net

/**

 * known bug: at resumptionType 'discover' if the (resultTotal % resultSizeValue) == 0 the collecting fails -> change the resultSizeValue

package eu.dnetlib.data.collector.plugins.rest;

import java.io.InputStream;

import java.nio.charset.StandardCharsets;

import java.util.Queue;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerConfigurationException;

import javax.xml.transform.TransformerFactory;

import javax.xml.transform.stream.StreamResult;

import org.apache.commons.logging.LogFactory;

/**

 */

public class RestIterator implements Iterator<String> {

	private String resumptionType;

	private String resumptionParam;

	private String resultFormatValue;

	private String queryParams;

	private String resumptionStr = Integer.toString(resumptionInt);  // string resumption token (first record to harvest or token scanned from results)

	private InputStream resultStream;

	private Transformer transformer;

	private XPath xpath;

	private XPathExpression xprResumptionPath;

	private String querySize;

	private String authToken;

	private int pagination = 1;

	 * RestIterator class

	 *

	 * compatible to version before 1.3.33

	 *

	 * @param baseUrl

	 * @param resumptionType

	 * @param resumptionParam

	 * @param resumptionXpath

	 * @param resultTotalXpath

	 * @param resultFormatParam

	 * @param resultFormatValue

	 * @param resultSizeParam

	 * @param resultSizeValueStr

	 * @param queryParams

	 * @param entityXpath

	 */

			final String baseUrl,

			final String resumptionType,

			final String resumptionParam,

			final String resumptionXpath,

			final String resultTotalXpath,

			final String resultFormatParam,

			final String resultFormatValue,

			final String resultSizeParam,

			final String entityXpath

			final String baseUrl,

			final String resumptionType,

			final String resumptionParam,

			final String resumptionXpath,

			final String resultTotalXpath,

			final String resultFormatParam,

			final String resultFormatValue,

			final String resultSizeParam,

			final String resultSizeValueStr,

			final String queryParams,

			final String entityXpath,

			final String authMethod,

			final String authToken,

			final String resultOffsetParam

	) {

		this(baseUrl,resumptionType,resumptionParam,resumptionXpath,resultTotalXpath,resultFormatParam,resultFormatValue,resultSizeParam,resultSizeValueStr,queryParams,entityXpath,"", "");

	}

	 *  compatible to version 1.3.33

	 * @param baseUrl

	 * @param resumptionType

	 * @param resumptionParam

	 * @param resumptionXpath

	 * @param resultTotalXpath

	 * @param resultFormatParam

	 * @param resultFormatValue

	 * @param resultSizeParam

	 * @param resultSizeValueStr

	 * @param queryParams

	 * @param entityXpath

	 * @param authMethod

	 * @param authToken

	 */

			final String baseUrl,

			final String resumptionType,

			final String resumptionParam,

			final String resumptionXpath,

			final String resultTotalXpath,

			final String resultFormatParam,

			final String resultFormatValue,

			final String resultSizeParam,

			final String resultSizeValueStr,

			final String queryParams,

			final String entityXpath,

			final String authMethod,

			final String authToken

	) {

		this.resumptionType = resumptionType;

		this.resumptionParam = resumptionParam;

		this.resultFormatValue = resultFormatValue;

		this.queryParams = queryParams;

		this.authToken = authToken;

		querySize = StringUtils.isNotBlank(resultSizeParam) ? "&" + resultSizeParam + "=" + resultSizeValueStr : "";

			throws TransformerConfigurationException, XPathExpressionException {

		transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "3");

		xpath = XPathFactory.newInstance().newXPath();

		xprEntity = xpath.compile(entityXpath);

		// TODO close inputstream

	}

	 * @see java.util.Iterator#hasNext()

	 */

	@Override

	public boolean hasNext() {

			return false;

		} else {

			return true;

		}

	}

	/* (non-Javadoc)

	 * @see java.util.Iterator#next()

	 */

	@Override

	public String next() {

				} catch (CollectorServiceException e) {

					log.debug("CollectorPlugin.next()-Exception: " + e);

				}

			}

			return recordQueue.poll();

		}

	}

	 * download page and return nextQuery

	 */

		NodeList nodeList = null;

		String qUrlArgument = "";

		int urlOldResumptionSize = 0;

		// check if cursor=* is initial set otherwise add it to the queryParam URL

		if( resumptionType.equalsIgnoreCase("deep-cursor") ) {

			log.debug("check resumptionType deep-cursor and check cursor=*?" + query);

			if(!query.contains("&cursor=")) {

				query += "&cursor=*";

			}

		}

	        	conn.setRequestProperty("Authorization","Bearer "+authToken);

	        	conn.setRequestProperty("Content-Type","application/json");

				conn.setRequestProperty("Authorization","Basic "+authToken);

				conn.setRequestProperty("accept","application/xml");

				conn.setRequestMethod("GET");

				log.debug("url : " + qUrl.getPath());

			}

			resultStream = theHttpInputStream;

				resultJson = IOUtils.toString(resultStream, "UTF-8");

			if (!(emptyXml.toLowerCase()).equals(resultXml.toLowerCase())) {

				resultNode = (Node) xpath.evaluate("/", new InputSource(resultStream), XPathConstants.NODE);

				nodeList = (NodeList) xprEntity.evaluate(resultNode, XPathConstants.NODESET);

				log.debug("nodeList.length: " + nodeList.getLength());

				for (int i = 0; i < nodeList.getLength(); i++) {

					StringWriter sw = new StringWriter();

					transformer.transform(new DOMSource(nodeList.item(i)), new StreamResult(sw));

					recordQueue.add(sw.toString());

				}

			} else { log.info("resultXml is equal with emptyXml"); }

			case "scan":    // read of resumptionToken , evaluate next results, e.g. OAI, iterate over items

				resumptionStr = xprResumptionPath.evaluate(resultNode);

				break;

				resumptionStr = Integer.toString(resumptionInt);

				break;

			case "discover":   // size of result items unknown, iterate over items  (for openDOAR - 201808)

				if (resultSizeValue < 2) {throw new CollectorServiceException("Mode: discover, Param 'resultSizeValue' is less than 2");}

				qUrlArgument = qUrl.getQuery();

				String[] arrayQUrlArgument = qUrlArgument.split("&");

				for (String arrayUrlArgStr : arrayQUrlArgument) {

					if (arrayUrlArgStr.startsWith(resumptionParam)) {

						String[] resumptionKeyValue = arrayUrlArgStr.split("=");

							urlOldResumptionSize = Integer.parseInt(resumptionKeyValue[1]);

							log.debug("discover OldResumptionSize from Url (int): " + urlOldResumptionSize);

						} else {

							log.debug("discover OldResumptionSize from Url (str): " + resumptionKeyValue[1]);

						}

				}

				if (((emptyXml.toLowerCase()).equals(resultXml.toLowerCase()))

						|| ((nodeList != null) && (nodeList.getLength() < resultSizeValue))

				) {

					// resumptionStr = "";

					if (nodeList != null) { discoverResultSize += nodeList.getLength(); }

					resultTotal = discoverResultSize;

				} else {

					resumptionStr = Integer.toString(resumptionInt);

					resultTotal = resumptionInt + 1;

					if (nodeList != null) { discoverResultSize += nodeList.getLength(); }

				}

			case "pagination":

				if (nodeList != null) {

					discoverResultSize += nodeList.getLength();

				} else {

					resultTotal = discoverResultSize;

					pagination = discoverResultSize;

				}

				resumptionInt = pagination;

				resumptionStr = Integer.toString(resumptionInt);

				break;

					resumptionInt += ( nodeList.getLength() + 1 - resultSizeValue);

					resumptionInt += (nodeList.getLength() - resultSizeValue);	// subtract the resultSizeValue because the iteration is over real length and the resultSizeValue is added before the switch()

				}

				discoverResultSize = nodeList.getLength();

				break;

				// resultTotal = resumptionInt;

				break;

			}

		} catch (Exception e) {

			throw new IllegalStateException("collection failed: " + e.getMessage());

			if (resultTotal == -1) {

				resultTotal = Integer.parseInt(xprResultTotalPath.evaluate(resultNode));

				if (resumptionType.toLowerCase().equals("page")) { resultTotal += 1; }           // to correct the upper bound

				log.info("resultTotal was -1 is now: " + resultTotal);

			log.error(e);

			throw new IllegalStateException("downloadPage resultTotal couldn't parse: " + e.getMessage());

		}

		log.debug("resInt: " + resumptionInt);

			nextQuery = baseUrl + "?" + queryParams + querySize + "&" + resumptionParam + "=" + resumptionStr + queryFormat;

		return nextQuery;

	private boolean isInteger(String s) {

		boolean isValidInteger = false;

		try {

			Integer.parseInt(s);

			isValidInteger = true;

		} catch (NumberFormatException ex) {

			// s is not an integer

		}

		return isValidInteger;

	}

	// Method to encode a string value using `UTF-8` encoding scheme

    private String encodeValue(String value) {

        try {

            return URLEncoder.encode(value, StandardCharsets.UTF_8.toString());

        } catch (UnsupportedEncodingException ex) {

            throw new RuntimeException(ex.getCause());

        }

    }