D-Net

 * 

 * 

 * 

 * known bug: at resumptionType 'discover' if the (resultTotal % resultSizeValue) == 0 the collecting fails -> change the resultSizeValue  

import javax.xml.xpath.XPath;

import javax.xml.xpath.XPathConstants;

import javax.xml.xpath.XPathExpression;

import javax.xml.xpath.XPathExpressionException;

import javax.xml.xpath.XPathFactory;

import eu.dnetlib.data.collector.plugins.oai.OaiIterator;

import eu.dnetlib.data.collector.plugins.oai.engine.XmlCleaner;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

    // TODO: clean up the comments of replaced source code

	private int resumptionInt = 0;			// integer resumption token (first record to harvest)

        private int discoverResultSize = 0;

        private int pagination = 1;

                        final String resultSizeValueStr,

			) {

                this.resultSizeValue = Integer.valueOf(resultSizeValueStr);

        queryFormat = StringUtils.isNotBlank(resultFormatParam) ? "&" + resultFormatParam + "=" + resultFormatValue : "";

        querySize = StringUtils.isNotBlank(resultSizeParam) ? "&" + resultSizeParam + "=" + resultSizeValueStr : "";

		} catch(Exception e) {

        initQueue();

	private void initXmlTransformation(String resultTotalXpath, String resumptionXpath, String entityXpath) throws TransformerConfigurationException, XPathExpressionException{

                transformer.setOutputProperty(OutputKeys.INDENT,"yes"); 

                transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount","3");

		xpath              = XPathFactory.newInstance().newXPath();

		xprResumptionPath  = xpath.compile(StringUtils.isBlank(resumptionXpath) ? "/" : resumptionXpath);

		xprEntity          = xpath.compile(entityXpath);

			while (recordQueue.isEmpty() && !query.isEmpty() ) {

                                        log.info("get Query: " + query);

                                        log.debug("next queryURL from downloadPage(): " + query);

				} catch(CollectorServiceException e) {

                                        log.debug("CollectorPlugin.next()-Exception: " + e);

	private String downloadPage(String query) throws CollectorServiceException{

                String emptyXml = resultXml + "<"+wrapName+"></"+wrapName+">";

                Node resultNode = null;

                NodeList nodeList = null;

                String qUrlArgument = "";

                int urlOldResumptionSize = 0;

                        URL qUrl = new URL(query);

                        resultStream = qUrl.openStream();

			if("json".equals(resultFormatValue.toLowerCase())){	

				resultJson = IOUtils.toString(resultStream,"UTF-8");

                                resultXml += org.json.XML.toString(jsonObject,wrapName); // wrap xml in single root element

                                resultXml = XmlCleaner.cleanAllEntities(resultXml);

                                log.trace("after cleaning: " + resultXml);

				resultStream = IOUtils.toInputStream(resultXml,"UTF-8");

                        if (!(emptyXml.toLowerCase()).equals(resultXml.toLowerCase())) {

                            resultNode = (Node) xpath.evaluate("/", new InputSource(resultStream), XPathConstants.NODE);

                            nodeList   = (NodeList) xprEntity.evaluate(resultNode, XPathConstants.NODESET);

                            log.debug("nodeList.length: " + nodeList.getLength());

                            for (int i = 0; i < nodeList.getLength(); i++) {

                                StringWriter sw = new StringWriter();

                                transformer.transform(new DOMSource(nodeList.item(i)), new StreamResult(sw));

                                recordQueue.add(sw.toString());

                            }

                        } else { log.info("resultXml is equal with emptyXml"); }	

                        switch(resumptionType.toLowerCase()) {

                            case "scan":    // read of resumptionToken , evaluate next results, e.g. OAI, iterate over items

                                            resumptionStr = xprResumptionPath.evaluate(resultNode);

                                            break;

                            case "count":   // begin at one step for all records, iterate over items

                                            resumptionStr = Integer.toString(resumptionInt);

                                            break;

                            case "discover":   // size of result items unknown, iterate over items  (for openDOAR - 201808)                                   

                                            if (resultSizeValue < 2 ) {throw new CollectorServiceException("Mode: discover, Param 'resultSizeValue' is less than 2");}

                                            qUrlArgument = qUrl.getQuery();

                                            String[] arrayQUrlArgument = qUrlArgument.split("&");

                                            for(String arrayUrlArgStr : arrayQUrlArgument ) {

                                                if(arrayUrlArgStr.startsWith(resumptionParam)) {

                                                    String[] resumptionKeyValue = arrayUrlArgStr.split("=");

                                                    urlOldResumptionSize = Integer.parseInt(resumptionKeyValue[1]);

                                                    log.debug("discover OldResumptionSize from Url: " + urlOldResumptionSize);

                                                }

                                            }

                                            if(   ( (emptyXml.toLowerCase()).equals(resultXml.toLowerCase()) ) 

                                               || ( (nodeList != null) && (nodeList.getLength() < resultSizeValue) ) 

                                              ) {

                                                // resumptionStr = "";

                                                if(nodeList != null) { discoverResultSize += nodeList.getLength(); }

                                                resultTotal   = discoverResultSize;

                                            } else {

                                                resumptionStr = Integer.toString(resumptionInt);

                                                resultTotal   = resumptionInt+1;

                                                if(nodeList != null) { discoverResultSize += nodeList.getLength(); }

                                            }

                                            log.info("discoverResultSize:  " + discoverResultSize);

                                            break;

                            case "pagination":   

                            case "page":         // pagination, iterate over pages

                                            pagination += 1;

                                            if (nodeList != null) {

                                                discoverResultSize += nodeList.getLength();

                                            } else {

                                                resultTotal = discoverResultSize;

                                                pagination  = discoverResultSize;

                                            }

                                            resumptionInt = pagination;

                                            resumptionStr = Integer.toString(resumptionInt);

                                            break;

                            default:        // otherwise: abort

                                            // resultTotal = resumptionInt;

                                            break;

                        }

                                if(resumptionType.toLowerCase().equals("page")) { resultTotal += 1; }           // to correct the upper bound

                        log.debug("nextQueryUrl: " + nextQuery);

		} catch(Exception e) {

        /**

         * convert in JSON-KeyName 'whitespace(s)' to '_' and '/' to '_', '(' and ')' to ''

         * check W3C XML syntax: https://www.w3.org/TR/2006/REC-xml11-20060816/#sec-starttags for valid tag names

         * and work-around for the JSON to XML converting of org.json.XML-package.

         * 

         * known bugs:     doesn't prevent     "key name":" ["sexy name",": penari","erotic dance"],

         * 

         * @param jsonInput

         * @return convertedJsonKeynameOutput

         */

        private String syntaxConvertJsonKeyNamens(String jsonInput) {

            log.trace("before convertJsonKeyNames: " + jsonInput);

            // pre-clean json - rid spaces of element names (misinterpreted as elements with attributes in xml)

            // replace ' 's in JSON Namens with '_'

            while (jsonInput.matches(".*\"([^\"]*)\\s+([^\"]*)\":.*")) {

                jsonInput = jsonInput.replaceAll("\"([^\"]*)\\s+([^\"]*)\":", "\"$1_$2\":");

            }

            // replace forward-slash (sign '/' ) in JSON Names with '_'

            while (jsonInput.matches(".*\"([^\"]*)/([^\"]*)\":.*")) {

                jsonInput = jsonInput.replaceAll("\"([^\"]*)/([^\"]*)\":", "\"$1_$2\":");

            }

            // replace '(' in JSON Names with ''

            while (jsonInput.matches(".*\"([^\"]*)[(]([^\"]*)\":.*")) {

                jsonInput = jsonInput.replaceAll("\"([^\"]*)[(]([^\"]*)\":", "\"$1$2\":");

            }

            // replace ')' in JSON Names with ''

            while (jsonInput.matches(".*\"([^\"]*)[)]([^\"]*)\":.*")) {

                jsonInput = jsonInput.replaceAll("\"([^\"]*)[)]([^\"]*)\":", "\"$1$2\":");

            }

            // replace startNumbers in JSON Keynames with 'n_'

            while (jsonInput.matches(".*\"([^\"][0-9])([^\"]*)\":.*")) {

                jsonInput = jsonInput.replaceAll("\"([^\"][0-9])([^\"]*)\":", "\"n_$1$2\":");

            }

            // replace ':' between number like '2018-08-28T11:05:00Z' in JSON keynames with ''

            while (jsonInput.matches(".*\"([^\"]*[0-9]):([0-9][^\"]*)\":.*")) {

                jsonInput = jsonInput.replaceAll("\"([^\"]*[0-9]):([0-9][^\"]*)\":", "\"$1$2\":");

            }            

            // replace ',' in JSON Keynames with '.' to prevent , in xml tagnames. 

//            while (jsonInput.matches(".*\"([^\"]*),([^\"]*)\":.*")) {

//                jsonInput = jsonInput.replaceAll("\"([^\"]*),([^\"]*)\":", "\"$1.$2\":");

//            }

            // replace '=' in JSON Keynames with '-'

            while (jsonInput.matches(".*\"([^\"]*)=([^\"]*)\":.*")) {

                jsonInput = jsonInput.replaceAll("\"([^\"]*)=([^\"]*)\":", "\"$1-$2\":");

            }

            log.trace("after syntaxConvertJsonKeyNames: " + jsonInput);

            return jsonInput;

        }

        /**

         * 

         * https://www.w3.org/TR/REC-xml/#charencoding shows character enoding in entities

         *          * 

         * @param bufferStr - XML string

         * @return 

         */

        private static String cleanUnwantedJsonCharsInXmlTagnames( String bufferStr ) {

            while ( bufferStr.matches(".*<([^<>].*),(.)>.*") ) {

                bufferStr = bufferStr.replaceAll("<([^<>.*),(.*)>", "<$1$2>");

            }

            // replace [#x10-#x1f] with ''

//            while (bufferStr.matches(".*&#x1[0-9a-f].*")) {

//                bufferStr = bufferStr.replaceAll("&#x1([0-9a-fA-F])", "");

//            }

            return bufferStr;

        }