Revision 58456
Added by Andreas Czerniak about 4 years ago
RestIterator.java | ||
---|---|---|
7 | 7 |
|
8 | 8 |
import java.io.InputStream; |
9 | 9 |
import java.io.StringWriter; |
10 |
import java.io.UnsupportedEncodingException; |
|
10 | 11 |
import java.net.URL; |
12 |
import java.net.URLEncoder; |
|
13 |
import java.nio.charset.StandardCharsets; |
|
11 | 14 |
import java.net.HttpURLConnection; |
12 | 15 |
import java.util.Iterator; |
13 | 16 |
import java.util.Queue; |
... | ... | |
32 | 35 |
|
33 | 36 |
/** |
34 | 37 |
* @author Jochen Schirrwagen, Aenne Loehden, Andreas Czerniak |
35 |
* @date 2020-03-20
|
|
38 |
* @date 2020-04-09
|
|
36 | 39 |
* |
37 | 40 |
*/ |
38 | 41 |
public class RestIterator implements Iterator<String> { |
... | ... | |
64 | 67 |
private Queue<String> recordQueue = new PriorityBlockingQueue<String>(); |
65 | 68 |
private int discoverResultSize = 0; |
66 | 69 |
private int pagination = 1; |
67 |
private String resultOffsetParam; |
|
68 | 70 |
|
69 | 71 |
/** |
70 | 72 |
* RestIterator class |
... | ... | |
96 | 98 |
final String queryParams, |
97 | 99 |
final String entityXpath |
98 | 100 |
) { |
99 |
this(baseUrl,resumptionType,resumptionParam,resumptionXpath,resultTotalXpath,resultFormatParam,resultFormatValue,resultSizeParam,resultSizeValueStr,queryParams,entityXpath,"", "","");
|
|
101 |
this(baseUrl,resumptionType,resumptionParam,resumptionXpath,resultTotalXpath,resultFormatParam,resultFormatValue,resultSizeParam,resultSizeValueStr,queryParams,entityXpath,"", ""); |
|
100 | 102 |
} |
101 | 103 |
|
102 | 104 |
/** RestIterator class |
... | ... | |
130 | 132 |
final String authMethod, |
131 | 133 |
final String authToken |
132 | 134 |
) { |
133 |
this(baseUrl,resumptionType,resumptionParam,resumptionXpath,resultTotalXpath,resultFormatParam,resultFormatValue,resultSizeParam,resultSizeValueStr,queryParams,entityXpath,"", "",""); |
|
134 |
} |
|
135 |
|
|
136 |
/** |
|
137 |
* |
|
138 |
* @param baseUrl |
|
139 |
* @param resumptionType |
|
140 |
* @param resumptionParam |
|
141 |
* @param resumptionXpath |
|
142 |
* @param resultTotalXpath |
|
143 |
* @param resultFormatParam |
|
144 |
* @param resultFormatValue |
|
145 |
* @param resultSizeParam |
|
146 |
* @param resultSizeValueStr |
|
147 |
* @param queryParams |
|
148 |
* @param entityXpath |
|
149 |
* @param authMethod |
|
150 |
* @param authToken |
|
151 |
* @param resultOffsetParam |
|
152 |
*/ |
|
153 |
public RestIterator( |
|
154 |
final String baseUrl, |
|
155 |
final String resumptionType, |
|
156 |
final String resumptionParam, |
|
157 |
final String resumptionXpath, |
|
158 |
final String resultTotalXpath, |
|
159 |
final String resultFormatParam, |
|
160 |
final String resultFormatValue, |
|
161 |
final String resultSizeParam, |
|
162 |
final String resultSizeValueStr, |
|
163 |
final String queryParams, |
|
164 |
final String entityXpath, |
|
165 |
final String authMethod, |
|
166 |
final String authToken, |
|
167 |
final String resultOffsetParam |
|
168 |
) { |
|
169 | 135 |
this.baseUrl = baseUrl; |
170 | 136 |
this.resumptionType = resumptionType; |
171 | 137 |
this.resumptionParam = resumptionParam; |
... | ... | |
174 | 140 |
this.resultSizeValue = Integer.valueOf(resultSizeValueStr); |
175 | 141 |
this.authMethod = authMethod; |
176 | 142 |
this.authToken = authToken; |
177 |
this.resultOffsetParam = resultOffsetParam; |
|
178 | 143 |
|
179 | 144 |
queryFormat = StringUtils.isNotBlank(resultFormatParam) ? "&" + resultFormatParam + "=" + resultFormatValue : ""; |
180 | 145 |
querySize = StringUtils.isNotBlank(resultSizeParam) ? "&" + resultSizeParam + "=" + resultSizeValueStr : ""; |
... | ... | |
348 | 313 |
|
349 | 314 |
case "deep-cursor": // size of result items unknown, iterate over items (for supporting deep cursor in solr) |
350 | 315 |
// isn't relevant -- if (resultSizeValue < 2) {throw new CollectorServiceException("Mode: deep-cursor, Param 'resultSizeValue' is less than 2");} |
351 |
// if ( (this.resultOffsetParam.length()) < 2) {throw new CollectorServiceException("Mode: deep-cursor, Param 'resultOffsetParam' is less than 2");} |
|
352 | 316 |
|
353 |
resumptionStr = xprResumptionPath.evaluate(resultNode);
|
|
317 |
resumptionStr = encodeValue(xprResumptionPath.evaluate(resultNode));
|
|
354 | 318 |
queryParams = queryParams.replace("&cursor=*", ""); |
355 |
// queryParams = queryParams.replace("&&", "&"); |
|
356 | 319 |
|
357 |
//resumptionStr += "&" + resultOffsetParam + "=" + Integer.toString(resumptionInt); |
|
358 |
|
|
320 |
resumptionInt += nodeList.getLength(); |
|
359 | 321 |
log.debug("downloadPage().deep-cursor: resumptionStr=" + resumptionStr + " ; queryParams=" + queryParams); |
360 | 322 |
|
361 | 323 |
break; |
... | ... | |
490 | 452 |
|
491 | 453 |
return isValidInteger; |
492 | 454 |
} |
455 |
|
|
456 |
// Method to encode a string value using `UTF-8` encoding scheme |
|
457 |
private String encodeValue(String value) { |
|
458 |
try { |
|
459 |
return URLEncoder.encode(value, StandardCharsets.UTF_8.toString()); |
|
460 |
} catch (UnsupportedEncodingException ex) { |
|
461 |
throw new RuntimeException(ex.getCause()); |
|
462 |
} |
|
463 |
} |
|
493 | 464 |
|
494 | 465 |
} |
Also available in: Unified diff
Rest-Collector plugin: changes deep-cursor for CrossRef, v1.3.39-SNAPSHOT