Revision 61481
Added by Andreas Czerniak over 2 years ago
| modules/dnet-collector-plugins/trunk/src/main/java/eu/dnetlib/data/collector/plugins/rest/RestIterator.java | ||
|---|---|---|
| 16 | 16 |
import java.util.Map; |
| 17 | 17 |
import java.util.Queue; |
| 18 | 18 |
import java.util.concurrent.PriorityBlockingQueue; |
| 19 |
import java.util.regex.Pattern; |
|
| 20 |
import java.util.regex.Matcher; |
|
| 19 | 21 |
import javax.xml.transform.OutputKeys; |
| 20 | 22 |
import javax.xml.transform.Transformer; |
| 21 | 23 |
import javax.xml.transform.TransformerConfigurationException; |
| ... | ... | |
| 198 | 200 |
int urlOldResumptionSize = 0; |
| 199 | 201 |
InputStream theHttpInputStream; |
| 200 | 202 |
|
| 203 |
// modifying request URL |
|
| 201 | 204 |
// check if cursor=* is initial set otherwise add it to the queryParam URL |
| 202 | 205 |
if (resumptionType.equalsIgnoreCase("deep-cursor")) {
|
| 203 | 206 |
log.debug("RestIterator.downloadPage():: check resumptionType deep-cursor and check cursor=*?" + query);
|
| ... | ... | |
| 205 | 208 |
query += "&cursor=*"; |
| 206 | 209 |
} |
| 207 | 210 |
} |
| 211 |
// find pagination page start number in queryParam and remove before start the first query |
|
| 212 |
if((resumptionType.toLowerCase().equals("pagination")) && (query.contains("paginationStart="))) {
|
|
| 208 | 213 |
|
| 214 |
final Matcher m = Pattern.compile("paginationStart=([0-9]+)").matcher(query);
|
|
| 215 |
m.find(); // guaranteed to be true for this regex |
|
| 216 |
|
|
| 217 |
String[] pageVal = m.group(0).split("=");
|
|
| 218 |
pagination = Integer.parseInt(pageVal[1]); |
|
| 219 |
|
|
| 220 |
// remove page start number from queryParams |
|
| 221 |
query = query.replaceFirst("&?paginationStart=[0-9]+", "");
|
|
| 222 |
|
|
| 223 |
} |
|
| 224 |
|
|
| 225 |
|
|
| 209 | 226 |
try {
|
| 227 |
|
|
| 210 | 228 |
URL qUrl = new URL(query); |
| 211 | 229 |
log.debug("authMethod :" + authMethod);
|
| 212 | 230 |
if (this.authMethod == "bearer") {
|
| ... | ... | |
| 307 | 325 |
|
| 308 | 326 |
case "pagination": |
| 309 | 327 |
case "page": // pagination, iterate over page numbers |
| 328 |
// find start page number |
|
| 310 | 329 |
pagination += 1; |
| 311 | 330 |
if (nodeList != null) {
|
| 312 | 331 |
discoverResultSize += nodeList.getLength(); |
| ... | ... | |
| 316 | 335 |
} |
| 317 | 336 |
resumptionInt = pagination; |
| 318 | 337 |
resumptionStr = Integer.toString(resumptionInt); |
| 338 |
|
|
| 339 |
log.debug("RestIterator.downloadPage().pagination:: resumptionStr=" + resumptionStr + " ; queryParams=" + queryParams + " ; resultTotal: " + resultTotal + " ; discoverResultSize: " + discoverResultSize);
|
|
| 340 |
|
|
| 319 | 341 |
break; |
| 320 | 342 |
|
| 321 | 343 |
case "deep-cursor": // size of result items unknown, iterate over items (for supporting deep cursor in solr) |
| ... | ... | |
| 349 | 371 |
|
| 350 | 372 |
try {
|
| 351 | 373 |
String resultTotalXpathEval = xprResultTotalPath.evaluate(resultNode); |
| 352 |
log.debug("resInt: " +resumptionInt + "; resultTotal: " + resultTotal + " ; resultTotalXpath eval.: " + resultTotalXpathEval);
|
|
| 374 |
|
|
| 375 |
log.debug("downloadPage():: resInt: " +resumptionInt + "; resultTotal: " + resultTotal + " ; resultTotalXpath eval.: " + resultTotalXpathEval + "/" + resultTotalXPathEval1);
|
|
| 353 | 376 |
if ((resultTotal == -1) && (!resultTotalXpathEval.isEmpty())) {
|
| 354 | 377 |
resultTotal = Integer.parseInt(xprResultTotalPath.evaluate(resultNode)); |
| 355 | 378 |
if (resumptionType.toLowerCase().equals("page") && !AUTHBASIC.equalsIgnoreCase(authMethod)) {
|
| ... | ... | |
| 361 | 384 |
log.error(e); |
| 362 | 385 |
throw new IllegalStateException("downloadPage() resultTotal couldn't parse: " + e.getMessage());
|
| 363 | 386 |
} |
| 364 |
log.debug("resultTotal: " + resultTotal);
|
|
| 365 |
log.debug("resInt: " + resumptionInt);
|
|
| 387 |
log.debug("resultTotal: " + resultTotal + " ; resInt: " + resumptionInt);
|
|
| 366 | 388 |
if (resumptionInt <= resultTotal) {
|
| 367 | 389 |
nextQuery = baseUrl + "?" + queryParams + querySize + "&" + resumptionParam + "=" + resumptionStr + queryFormat; |
| 368 | 390 |
} else {
|
Also available in: Unified diff
add feature to change the pagination start number, enhance debugging