Revision 61301
Added by Andreas Czerniak almost 3 years ago
RestIterator.java | ||
---|---|---|
145 | 145 |
query = baseUrl + "?" + queryParams + querySize + queryFormat; |
146 | 146 |
} |
147 | 147 |
|
148 |
log.info("REST calls starting with " + query); |
|
148 |
log.info("RestIterator.initQueue():: REST calls starting with " + query);
|
|
149 | 149 |
} |
150 | 150 |
|
151 | 151 |
private void disconnect() { |
... | ... | |
173 | 173 |
synchronized (recordQueue) { |
174 | 174 |
while (recordQueue.isEmpty() && !query.isEmpty()) { |
175 | 175 |
try { |
176 |
log.debug("get Query: " + query); |
|
176 |
log.debug("RestIterator.next():: get Query: " + query);
|
|
177 | 177 |
query = downloadPage(query); |
178 |
log.debug("next queryURL from downloadPage(): " + query); |
|
178 |
log.debug("RestIterator.next():: next queryURL from downloadPage(): " + query);
|
|
179 | 179 |
} catch (CollectorServiceException e) { |
180 |
log.debug("CollectorPlugin.next()-Exception: " + e); |
|
180 |
log.debug("RestIterator.next():: CollectorPlugin.next()-Exception: " + e);
|
|
181 | 181 |
throw new RuntimeException(e); |
182 | 182 |
} |
183 | 183 |
} |
... | ... | |
200 | 200 |
|
201 | 201 |
// check if cursor=* is initial set otherwise add it to the queryParam URL |
202 | 202 |
if (resumptionType.equalsIgnoreCase("deep-cursor")) { |
203 |
log.debug("check resumptionType deep-cursor and check cursor=*?" + query); |
|
203 |
log.debug("RestIterator.downloadPage():: check resumptionType deep-cursor and check cursor=*?" + query);
|
|
204 | 204 |
if (!query.contains("&cursor=")) { |
205 | 205 |
query += "&cursor=*"; |
206 | 206 |
} |
... | ... | |
210 | 210 |
URL qUrl = new URL(query); |
211 | 211 |
log.debug("authMethod :" + authMethod); |
212 | 212 |
if (this.authMethod == "bearer") { |
213 |
log.trace("authMethod before inputStream: " + resultXml); |
|
213 |
log.trace("RestIterator.downloadPage():: authMethod before inputStream: " + resultXml);
|
|
214 | 214 |
requestHeaders.put("Authorization", "Bearer " + authToken); |
215 |
requestHeaders.put("Content-Type", "application/json"); |
|
215 |
//requestHeaders.put("Content-Type", "application/json");
|
|
216 | 216 |
} else if (AUTHBASIC.equalsIgnoreCase(this.authMethod)) { |
217 |
log.trace("authMethod before inputStream: " + resultXml); |
|
217 |
log.trace("RestIterator.downloadPage():: authMethod before inputStream: " + resultXml);
|
|
218 | 218 |
requestHeaders.put("Authorization", "Basic " + authToken); |
219 |
requestHeaders.put("accept", "application/xml"); |
|
219 |
//requestHeaders.put("accept", "application/xml");
|
|
220 | 220 |
} |
221 | 221 |
|
222 | 222 |
HttpURLConnection conn = (HttpURLConnection) qUrl.openConnection(); |
... | ... | |
233 | 233 |
if (!isEmptyXml(resultXml)) { |
234 | 234 |
resultNode = (Node) xpath.evaluate("/", new InputSource(resultStream), XPathConstants.NODE); |
235 | 235 |
nodeList = (NodeList) xprEntity.evaluate(resultNode, XPathConstants.NODESET); |
236 |
log.debug("nodeList.length: " + nodeList.getLength());
|
|
236 |
log.debug("RestIterator.downloadPage():: nodeList.length=" + nodeList.getLength());
|
|
237 | 237 |
for (int i = 0; i < nodeList.getLength(); i++) { |
238 | 238 |
StringWriter sw = new StringWriter(); |
239 | 239 |
transformer.transform(new DOMSource(nodeList.item(i)), new StreamResult(sw)); |
240 | 240 |
String toEnqueue = sw.toString(); |
241 | 241 |
if (toEnqueue == null || StringUtils.isBlank(toEnqueue) || isEmptyXml(toEnqueue)) { |
242 |
log.warn("The following record resulted in empty item for the feeding queue: " + resultXml); |
|
242 |
log.warn("RestIterator.downloadPage():: The following record resulted in empty item for the feeding queue: " + resultXml);
|
|
243 | 243 |
} else { |
244 | 244 |
recordQueue.add(sw.toString()); |
245 | 245 |
} |
... | ... | |
261 | 261 |
|
262 | 262 |
case "discover": // size of result items unknown, iterate over items (for openDOAR - 201808) |
263 | 263 |
if (resultSizeValue < 2) { |
264 |
throw new CollectorServiceException("Mode: discover, Param 'resultSizeValue' is less than 2"); |
|
264 |
log.debug("RestIterator.downloadPage().discover:: ode: discover, Param 'resultSizeValue' must greater then 1"); |
|
265 |
throw new CollectorServiceException("Mode: discover, Param 'resultSizeValue' must greater then 1"); |
|
265 | 266 |
} |
266 |
log.debug("RestIterator.downloadPage/discover:: resumptionInt="+Integer.toString(resumptionInt)+"; "); |
|
267 |
log.debug("RestIterator.downloadPage/discover:: nodeList.length=" + nodeList.getLength()); |
|
267 |
log.debug("RestIterator.downloadPage().discover:: resumptionInt="+Integer.toString(resumptionInt)+"; "); |
|
268 | 268 |
qUrlArgument = qUrl.getQuery(); |
269 | 269 |
String[] arrayQUrlArgument = qUrlArgument.split("&"); |
270 |
for (String arrayUrlArgStr : arrayQUrlArgument) { |
|
271 |
log.debug("RestIterator.downloadPage/discover:: "+arrayUrlArgStr); |
|
272 |
if (arrayUrlArgStr.startsWith(resumptionParam)) { |
|
273 |
String[] resumptionKeyValue = arrayUrlArgStr.split("="); |
|
274 |
if (isInteger(resumptionKeyValue[1])) { |
|
275 |
urlOldResumptionSize = Integer.parseInt(resumptionKeyValue[1]); |
|
276 |
log.debug("discover OldResumptionSize from Url (int): " + urlOldResumptionSize); |
|
277 |
} else { |
|
278 |
log.debug("discover OldResumptionSize from Url (str): " + resumptionKeyValue[1]); |
|
270 |
|
|
271 |
// check if URL arguments given |
|
272 |
if( arrayQUrlArgument != null ) { |
|
273 |
for (String arrayUrlArgStr : arrayQUrlArgument) { |
|
274 |
log.debug("RestIterator.downloadPage/discover:: "+arrayUrlArgStr); |
|
275 |
if (arrayUrlArgStr.startsWith(resumptionParam)) { |
|
276 |
String[] resumptionKeyValue = arrayUrlArgStr.split("="); |
|
277 |
if (isInteger(resumptionKeyValue[1])) { |
|
278 |
urlOldResumptionSize = Integer.parseInt(resumptionKeyValue[1]); |
|
279 |
log.debug("RestIterator.downloadPage():discover:: OldResumptionSize from Url (int): " + urlOldResumptionSize); |
|
280 |
} else { |
|
281 |
log.debug("RestIterator.downloadPage().discover:: OldResumptionSize from Url (str): " + resumptionKeyValue[1]); |
|
282 |
} |
|
279 | 283 |
} |
280 | 284 |
} |
281 | 285 |
} |
286 |
log.debug("RestIterator.downloadPage().discover:: nodeList.length=" + nodeList.getLength()); |
|
282 | 287 |
|
283 | 288 |
if (isEmptyXml(resultXml) || ((nodeList != null) && (nodeList.getLength() < resultSizeValue)) |
284 | 289 |
) { |
... | ... | |
294 | 299 |
discoverResultSize += nodeList.getLength(); |
295 | 300 |
} |
296 | 301 |
} |
297 |
log.debug("discoverResultSize: " + discoverResultSize);
|
|
302 |
log.debug("RestIterator.downloadPage().discover:: discoverResultSize=" + discoverResultSize);
|
|
298 | 303 |
break; |
299 | 304 |
|
300 | 305 |
case "pagination": |
... | ... | |
325 | 330 |
|
326 | 331 |
discoverResultSize = nodeList.getLength(); |
327 | 332 |
|
328 |
log.debug("downloadPage().deep-cursor: resumptionStr=" + resumptionStr + " ; queryParams=" + queryParams + " resumptionLengthIncreased: " + resumptionInt);
|
|
333 |
log.debug("RestIterator.downloadPage().deep-cursor:: resumptionStr=" + resumptionStr + " ; queryParams=" + queryParams + " resumptionLengthIncreased: " + resumptionInt);
|
|
329 | 334 |
|
330 | 335 |
break; |
331 | 336 |
|
... | ... | |
394 | 399 |
} |
395 | 400 |
} |
396 | 401 |
|
402 |
/** |
|
403 |
* setRequestHeader |
|
404 |
* |
|
405 |
* setRequestProperty: Sets the general request property. If a property with the key already exists, overwrite its value with the new value. |
|
406 |
* @param conn |
|
407 |
*/ |
|
397 | 408 |
private void setRequestHeader(HttpURLConnection conn) { |
398 | 409 |
if (requestHeaders != null) { |
399 | 410 |
for (String key : requestHeaders.keySet()) { |
Also available in: Unified diff
to avoid null pointer exception