7 |
7 |
|
8 |
8 |
import java.io.InputStream;
|
9 |
9 |
import java.io.StringWriter;
|
|
10 |
import java.io.UnsupportedEncodingException;
|
10 |
11 |
import java.net.URL;
|
|
12 |
import java.net.URLEncoder;
|
|
13 |
import java.nio.charset.StandardCharsets;
|
11 |
14 |
import java.net.HttpURLConnection;
|
12 |
15 |
import java.util.Iterator;
|
13 |
16 |
import java.util.Queue;
|
... | ... | |
32 |
35 |
|
33 |
36 |
/**
|
34 |
37 |
* @author Jochen Schirrwagen, Aenne Loehden, Andreas Czerniak
|
35 |
|
* @date 2020-03-20
|
|
38 |
* @date 2020-04-09
|
36 |
39 |
*
|
37 |
40 |
*/
|
38 |
41 |
public class RestIterator implements Iterator<String> {
|
... | ... | |
64 |
67 |
private Queue<String> recordQueue = new PriorityBlockingQueue<String>();
|
65 |
68 |
private int discoverResultSize = 0;
|
66 |
69 |
private int pagination = 1;
|
67 |
|
private String resultOffsetParam;
|
68 |
70 |
|
69 |
71 |
/**
|
70 |
72 |
* RestIterator class
|
... | ... | |
96 |
98 |
final String queryParams,
|
97 |
99 |
final String entityXpath
|
98 |
100 |
) {
|
99 |
|
this(baseUrl,resumptionType,resumptionParam,resumptionXpath,resultTotalXpath,resultFormatParam,resultFormatValue,resultSizeParam,resultSizeValueStr,queryParams,entityXpath,"", "","");
|
|
101 |
this(baseUrl,resumptionType,resumptionParam,resumptionXpath,resultTotalXpath,resultFormatParam,resultFormatValue,resultSizeParam,resultSizeValueStr,queryParams,entityXpath,"", "");
|
100 |
102 |
}
|
101 |
103 |
|
102 |
104 |
/** RestIterator class
|
... | ... | |
130 |
132 |
final String authMethod,
|
131 |
133 |
final String authToken
|
132 |
134 |
) {
|
133 |
|
this(baseUrl,resumptionType,resumptionParam,resumptionXpath,resultTotalXpath,resultFormatParam,resultFormatValue,resultSizeParam,resultSizeValueStr,queryParams,entityXpath,"", "","");
|
134 |
|
}
|
135 |
|
|
136 |
|
/**
|
137 |
|
*
|
138 |
|
* @param baseUrl
|
139 |
|
* @param resumptionType
|
140 |
|
* @param resumptionParam
|
141 |
|
* @param resumptionXpath
|
142 |
|
* @param resultTotalXpath
|
143 |
|
* @param resultFormatParam
|
144 |
|
* @param resultFormatValue
|
145 |
|
* @param resultSizeParam
|
146 |
|
* @param resultSizeValueStr
|
147 |
|
* @param queryParams
|
148 |
|
* @param entityXpath
|
149 |
|
* @param authMethod
|
150 |
|
* @param authToken
|
151 |
|
* @param resultOffsetParam
|
152 |
|
*/
|
153 |
|
public RestIterator(
|
154 |
|
final String baseUrl,
|
155 |
|
final String resumptionType,
|
156 |
|
final String resumptionParam,
|
157 |
|
final String resumptionXpath,
|
158 |
|
final String resultTotalXpath,
|
159 |
|
final String resultFormatParam,
|
160 |
|
final String resultFormatValue,
|
161 |
|
final String resultSizeParam,
|
162 |
|
final String resultSizeValueStr,
|
163 |
|
final String queryParams,
|
164 |
|
final String entityXpath,
|
165 |
|
final String authMethod,
|
166 |
|
final String authToken,
|
167 |
|
final String resultOffsetParam
|
168 |
|
) {
|
169 |
135 |
this.baseUrl = baseUrl;
|
170 |
136 |
this.resumptionType = resumptionType;
|
171 |
137 |
this.resumptionParam = resumptionParam;
|
... | ... | |
174 |
140 |
this.resultSizeValue = Integer.valueOf(resultSizeValueStr);
|
175 |
141 |
this.authMethod = authMethod;
|
176 |
142 |
this.authToken = authToken;
|
177 |
|
this.resultOffsetParam = resultOffsetParam;
|
178 |
143 |
|
179 |
144 |
queryFormat = StringUtils.isNotBlank(resultFormatParam) ? "&" + resultFormatParam + "=" + resultFormatValue : "";
|
180 |
145 |
querySize = StringUtils.isNotBlank(resultSizeParam) ? "&" + resultSizeParam + "=" + resultSizeValueStr : "";
|
... | ... | |
348 |
313 |
|
349 |
314 |
case "deep-cursor": // size of result items unknown, iterate over items (for supporting deep cursor in solr)
|
350 |
315 |
// isn't relevant -- if (resultSizeValue < 2) {throw new CollectorServiceException("Mode: deep-cursor, Param 'resultSizeValue' is less than 2");}
|
351 |
|
// if ( (this.resultOffsetParam.length()) < 2) {throw new CollectorServiceException("Mode: deep-cursor, Param 'resultOffsetParam' is less than 2");}
|
352 |
316 |
|
353 |
|
resumptionStr = xprResumptionPath.evaluate(resultNode);
|
|
317 |
resumptionStr = encodeValue(xprResumptionPath.evaluate(resultNode));
|
354 |
318 |
queryParams = queryParams.replace("&cursor=*", "");
|
355 |
|
// queryParams = queryParams.replace("&&", "&");
|
356 |
319 |
|
357 |
|
//resumptionStr += "&" + resultOffsetParam + "=" + Integer.toString(resumptionInt);
|
358 |
|
|
|
320 |
resumptionInt += nodeList.getLength();
|
359 |
321 |
log.debug("downloadPage().deep-cursor: resumptionStr=" + resumptionStr + " ; queryParams=" + queryParams);
|
360 |
322 |
|
361 |
323 |
break;
|
... | ... | |
490 |
452 |
|
491 |
453 |
return isValidInteger;
|
492 |
454 |
}
|
|
455 |
|
|
456 |
// Method to encode a string value using `UTF-8` encoding scheme
|
|
457 |
private String encodeValue(String value) {
|
|
458 |
try {
|
|
459 |
return URLEncoder.encode(value, StandardCharsets.UTF_8.toString());
|
|
460 |
} catch (UnsupportedEncodingException ex) {
|
|
461 |
throw new RuntimeException(ex.getCause());
|
|
462 |
}
|
|
463 |
}
|
493 |
464 |
|
494 |
465 |
}
|
Rest-Collector plugin: changes deep-cursor for CrossRef, v1.3.39-SNAPSHOT