Revision 52983
Added by Andreas Czerniak over 5 years ago
RestIterator.java | ||
---|---|---|
1 | 1 |
/** |
2 | 2 |
* |
3 |
* |
|
4 |
* log.debug(...) equal to log.trace(...) in the application-logs |
|
3 | 5 |
*/ |
4 | 6 |
package eu.dnetlib.data.collector.plugins.rest; |
5 | 7 |
|
... | ... | |
63 | 65 |
private String queryFormat; |
64 | 66 |
private String querySize; |
65 | 67 |
private Queue<String> recordQueue = new PriorityBlockingQueue<String>(); |
66 |
|
|
68 |
private int discoverResultSize = 0; |
|
69 |
|
|
67 | 70 |
public RestIterator( |
68 | 71 |
final String baseUrl, |
69 | 72 |
final String resumptionType, |
... | ... | |
134 | 137 |
synchronized (recordQueue) { |
135 | 138 |
while (recordQueue.isEmpty() && !query.isEmpty() ) { |
136 | 139 |
try { |
140 |
log.info("get Query: " + query); |
|
137 | 141 |
query = downloadPage(query); |
142 |
log.debug("next query from downloadPage method: " + query); |
|
138 | 143 |
} catch(CollectorServiceException e) { |
144 |
log.debug("CollectorPlugin.next()-Exception: " + e); |
|
139 | 145 |
throw new RuntimeException(e); |
140 | 146 |
} |
141 | 147 |
} |
... | ... | |
154 | 160 |
String emptyXml = "<"+wrapName+"></"+wrapName+">"; |
155 | 161 |
Node resultNode = null; |
156 | 162 |
NodeList nodeList = null; |
157 |
int discoverResultSize = 0; |
|
163 |
String qUrlArgument = ""; |
|
164 |
int urlOldResumptionSize = 0; |
|
158 | 165 |
|
159 | 166 |
try { |
160 |
resultStream = new URL(query).openStream(); |
|
167 |
URL qUrl = new URL(query); |
|
168 |
|
|
169 |
resultStream = qUrl.openStream(); |
|
161 | 170 |
if("json".equals(resultFormatValue.toLowerCase())){ |
162 | 171 |
resultJson = IOUtils.toString(resultStream,"UTF-8"); |
163 | 172 |
|
... | ... | |
200 | 209 |
break; |
201 | 210 |
|
202 | 211 |
case "discover": // length of results unknown |
203 |
if( (emptyXml.toLowerCase()).equals(resultXml.toLowerCase()) ) { |
|
212 |
qUrlArgument = qUrl.getQuery(); |
|
213 |
String[] arrayQUrlArgument = qUrlArgument.split("&"); |
|
214 |
for(String arrayUrlArgStr : arrayQUrlArgument ) { |
|
215 |
if(arrayUrlArgStr.startsWith(resumptionParam)) { |
|
216 |
String[] resumptionKeyValue = arrayUrlArgStr.split("="); |
|
217 |
urlOldResumptionSize = Integer.parseInt(resumptionKeyValue[1]); |
|
218 |
log.debug("discover OldResumptionSize from Url: " + urlOldResumptionSize); |
|
219 |
} |
|
220 |
} |
|
221 |
|
|
222 |
// |
|
223 |
if( ( (emptyXml.toLowerCase()).equals(resultXml.toLowerCase()) ) |
|
224 |
|| ( (nodeList != null) && (nodeList.getLength() < resultSizeValue) ) |
|
225 |
) { |
|
204 | 226 |
// resumptionStr = ""; |
227 |
if(nodeList != null) { discoverResultSize += nodeList.getLength(); } |
|
205 | 228 |
resultTotal = discoverResultSize; |
206 | 229 |
} else { |
207 | 230 |
resumptionStr = Integer.toString(resumptionInt); |
Also available in: Unified diff
Additional to discover option in the Rest_Json CollectorPlugin for the enhancements of the new OpenDOAR API at JISC under https://v2.sherpa.ac.uk/opendoar/