| 145 |
145 |
query = baseUrl + "?" + queryParams + querySize + queryFormat;
|
| 146 |
146 |
}
|
| 147 |
147 |
|
| 148 |
|
log.info("REST calls starting with " + query);
|
|
148 |
log.info("RestIterator.initQueue():: REST calls starting with " + query);
|
| 149 |
149 |
}
|
| 150 |
150 |
|
| 151 |
151 |
private void disconnect() {
|
| ... | ... | |
| 173 |
173 |
synchronized (recordQueue) {
|
| 174 |
174 |
while (recordQueue.isEmpty() && !query.isEmpty()) {
|
| 175 |
175 |
try {
|
| 176 |
|
log.debug("get Query: " + query);
|
|
176 |
log.debug("RestIterator.next():: get Query: " + query);
|
| 177 |
177 |
query = downloadPage(query);
|
| 178 |
|
log.debug("next queryURL from downloadPage(): " + query);
|
|
178 |
log.debug("RestIterator.next():: next queryURL from downloadPage(): " + query);
|
| 179 |
179 |
} catch (CollectorServiceException e) {
|
| 180 |
|
log.debug("CollectorPlugin.next()-Exception: " + e);
|
|
180 |
log.debug("RestIterator.next():: CollectorPlugin.next()-Exception: " + e);
|
| 181 |
181 |
throw new RuntimeException(e);
|
| 182 |
182 |
}
|
| 183 |
183 |
}
|
| ... | ... | |
| 200 |
200 |
|
| 201 |
201 |
// check if cursor=* is initial set otherwise add it to the queryParam URL
|
| 202 |
202 |
if (resumptionType.equalsIgnoreCase("deep-cursor")) {
|
| 203 |
|
log.debug("check resumptionType deep-cursor and check cursor=*?" + query);
|
|
203 |
log.debug("RestIterator.downloadPage():: check resumptionType deep-cursor and check cursor=*?" + query);
|
| 204 |
204 |
if (!query.contains("&cursor=")) {
|
| 205 |
205 |
query += "&cursor=*";
|
| 206 |
206 |
}
|
| ... | ... | |
| 210 |
210 |
URL qUrl = new URL(query);
|
| 211 |
211 |
log.debug("authMethod :" + authMethod);
|
| 212 |
212 |
if (this.authMethod == "bearer") {
|
| 213 |
|
log.trace("authMethod before inputStream: " + resultXml);
|
|
213 |
log.trace("RestIterator.downloadPage():: authMethod before inputStream: " + resultXml);
|
| 214 |
214 |
requestHeaders.put("Authorization", "Bearer " + authToken);
|
| 215 |
|
requestHeaders.put("Content-Type", "application/json");
|
|
215 |
//requestHeaders.put("Content-Type", "application/json");
|
| 216 |
216 |
} else if (AUTHBASIC.equalsIgnoreCase(this.authMethod)) {
|
| 217 |
|
log.trace("authMethod before inputStream: " + resultXml);
|
|
217 |
log.trace("RestIterator.downloadPage():: authMethod before inputStream: " + resultXml);
|
| 218 |
218 |
requestHeaders.put("Authorization", "Basic " + authToken);
|
| 219 |
|
requestHeaders.put("accept", "application/xml");
|
|
219 |
//requestHeaders.put("accept", "application/xml");
|
| 220 |
220 |
}
|
| 221 |
221 |
|
| 222 |
222 |
HttpURLConnection conn = (HttpURLConnection) qUrl.openConnection();
|
| ... | ... | |
| 233 |
233 |
if (!isEmptyXml(resultXml)) {
|
| 234 |
234 |
resultNode = (Node) xpath.evaluate("/", new InputSource(resultStream), XPathConstants.NODE);
|
| 235 |
235 |
nodeList = (NodeList) xprEntity.evaluate(resultNode, XPathConstants.NODESET);
|
| 236 |
|
log.debug("nodeList.length: " + nodeList.getLength());
|
|
236 |
log.debug("RestIterator.downloadPage():: nodeList.length=" + nodeList.getLength());
|
| 237 |
237 |
for (int i = 0; i < nodeList.getLength(); i++) {
|
| 238 |
238 |
StringWriter sw = new StringWriter();
|
| 239 |
239 |
transformer.transform(new DOMSource(nodeList.item(i)), new StreamResult(sw));
|
| 240 |
240 |
String toEnqueue = sw.toString();
|
| 241 |
241 |
if (toEnqueue == null || StringUtils.isBlank(toEnqueue) || isEmptyXml(toEnqueue)) {
|
| 242 |
|
log.warn("The following record resulted in empty item for the feeding queue: " + resultXml);
|
|
242 |
log.warn("RestIterator.downloadPage():: The following record resulted in empty item for the feeding queue: " + resultXml);
|
| 243 |
243 |
} else {
|
| 244 |
244 |
recordQueue.add(sw.toString());
|
| 245 |
245 |
}
|
| ... | ... | |
| 261 |
261 |
|
| 262 |
262 |
case "discover": // size of result items unknown, iterate over items (for openDOAR - 201808)
|
| 263 |
263 |
if (resultSizeValue < 2) {
|
| 264 |
|
throw new CollectorServiceException("Mode: discover, Param 'resultSizeValue' is less than 2");
|
|
264 |
log.debug("RestIterator.downloadPage().discover:: ode: discover, Param 'resultSizeValue' must greater then 1");
|
|
265 |
throw new CollectorServiceException("Mode: discover, Param 'resultSizeValue' must greater then 1");
|
| 265 |
266 |
}
|
| 266 |
|
log.debug("RestIterator.downloadPage/discover:: resumptionInt="+Integer.toString(resumptionInt)+"; ");
|
| 267 |
|
log.debug("RestIterator.downloadPage/discover:: nodeList.length=" + nodeList.getLength());
|
|
267 |
log.debug("RestIterator.downloadPage().discover:: resumptionInt="+Integer.toString(resumptionInt)+"; ");
|
| 268 |
268 |
qUrlArgument = qUrl.getQuery();
|
| 269 |
269 |
String[] arrayQUrlArgument = qUrlArgument.split("&");
|
| 270 |
|
for (String arrayUrlArgStr : arrayQUrlArgument) {
|
| 271 |
|
log.debug("RestIterator.downloadPage/discover:: "+arrayUrlArgStr);
|
| 272 |
|
if (arrayUrlArgStr.startsWith(resumptionParam)) {
|
| 273 |
|
String[] resumptionKeyValue = arrayUrlArgStr.split("=");
|
| 274 |
|
if (isInteger(resumptionKeyValue[1])) {
|
| 275 |
|
urlOldResumptionSize = Integer.parseInt(resumptionKeyValue[1]);
|
| 276 |
|
log.debug("discover OldResumptionSize from Url (int): " + urlOldResumptionSize);
|
| 277 |
|
} else {
|
| 278 |
|
log.debug("discover OldResumptionSize from Url (str): " + resumptionKeyValue[1]);
|
|
270 |
|
|
271 |
// check if URL arguments given
|
|
272 |
if( arrayQUrlArgument != null ) {
|
|
273 |
for (String arrayUrlArgStr : arrayQUrlArgument) {
|
|
274 |
log.debug("RestIterator.downloadPage/discover:: "+arrayUrlArgStr);
|
|
275 |
if (arrayUrlArgStr.startsWith(resumptionParam)) {
|
|
276 |
String[] resumptionKeyValue = arrayUrlArgStr.split("=");
|
|
277 |
if (isInteger(resumptionKeyValue[1])) {
|
|
278 |
urlOldResumptionSize = Integer.parseInt(resumptionKeyValue[1]);
|
|
279 |
log.debug("RestIterator.downloadPage():discover:: OldResumptionSize from Url (int): " + urlOldResumptionSize);
|
|
280 |
} else {
|
|
281 |
log.debug("RestIterator.downloadPage().discover:: OldResumptionSize from Url (str): " + resumptionKeyValue[1]);
|
|
282 |
}
|
| 279 |
283 |
}
|
| 280 |
284 |
}
|
| 281 |
285 |
}
|
|
286 |
log.debug("RestIterator.downloadPage().discover:: nodeList.length=" + nodeList.getLength());
|
| 282 |
287 |
|
| 283 |
288 |
if (isEmptyXml(resultXml) || ((nodeList != null) && (nodeList.getLength() < resultSizeValue))
|
| 284 |
289 |
) {
|
| ... | ... | |
| 294 |
299 |
discoverResultSize += nodeList.getLength();
|
| 295 |
300 |
}
|
| 296 |
301 |
}
|
| 297 |
|
log.debug("discoverResultSize: " + discoverResultSize);
|
|
302 |
log.debug("RestIterator.downloadPage().discover:: discoverResultSize=" + discoverResultSize);
|
| 298 |
303 |
break;
|
| 299 |
304 |
|
| 300 |
305 |
case "pagination":
|
| ... | ... | |
| 325 |
330 |
|
| 326 |
331 |
discoverResultSize = nodeList.getLength();
|
| 327 |
332 |
|
| 328 |
|
log.debug("downloadPage().deep-cursor: resumptionStr=" + resumptionStr + " ; queryParams=" + queryParams + " resumptionLengthIncreased: " + resumptionInt);
|
|
333 |
log.debug("RestIterator.downloadPage().deep-cursor:: resumptionStr=" + resumptionStr + " ; queryParams=" + queryParams + " resumptionLengthIncreased: " + resumptionInt);
|
| 329 |
334 |
|
| 330 |
335 |
break;
|
| 331 |
336 |
|
| ... | ... | |
| 394 |
399 |
}
|
| 395 |
400 |
}
|
| 396 |
401 |
|
|
402 |
/**
|
|
403 |
* setRequestHeader
|
|
404 |
*
|
|
405 |
* setRequestProperty: Sets the general request property. If a property with the key already exists, overwrite its value with the new value.
|
|
406 |
* @param conn
|
|
407 |
*/
|
| 397 |
408 |
private void setRequestHeader(HttpURLConnection conn) {
|
| 398 |
409 |
if (requestHeaders != null) {
|
| 399 |
410 |
for (String key : requestHeaders.keySet()) {
|
Added by Andreas Czerniak almost 3 years ago
to avoid null pointer exception