Revision 52983
Added by Andreas Czerniak over 5 years ago
| modules/dnet-collector-plugins/trunk/src/main/java/eu/dnetlib/data/collector/plugins/rest/RestCollectorPlugin.java | ||
|---|---|---|
| 38 | 38 |
//if (StringUtils.isBlank(resultFormatParam)) {throw new CollectorServiceException("Param 'resultFormatParam' is null, empty or whitespace");}
|
| 39 | 39 |
if (StringUtils.isBlank(resultFormatValue)) {throw new CollectorServiceException("Param 'resultFormatValue' is null or empty");}
|
| 40 | 40 |
if (StringUtils.isBlank(resultSizeParam)) {throw new CollectorServiceException("Param 'resultSizeParam' is null or empty");}
|
| 41 |
if (Integer.valueOf(resultSizeValue) <= 1) {throw new CollectorServiceException("Param 'resultSizeValue' is less than 2");}
|
|
| 41 | 42 |
if (StringUtils.isBlank(queryParams)) {throw new CollectorServiceException("Param 'queryParams' is null or empty");}
|
| 42 | 43 |
if (StringUtils.isBlank(entityXpath)) {throw new CollectorServiceException("Param 'entityXpath' is null or empty");}
|
| 43 | 44 |
|
| modules/dnet-collector-plugins/trunk/src/main/java/eu/dnetlib/data/collector/plugins/rest/RestIterator.java | ||
|---|---|---|
| 1 | 1 |
/** |
| 2 | 2 |
* |
| 3 |
* |
|
| 4 |
* log.debug(...) equal to log.trace(...) in the application-logs |
|
| 3 | 5 |
*/ |
| 4 | 6 |
package eu.dnetlib.data.collector.plugins.rest; |
| 5 | 7 |
|
| ... | ... | |
| 63 | 65 |
private String queryFormat; |
| 64 | 66 |
private String querySize; |
| 65 | 67 |
private Queue<String> recordQueue = new PriorityBlockingQueue<String>(); |
| 66 |
|
|
| 68 |
private int discoverResultSize = 0; |
|
| 69 |
|
|
| 67 | 70 |
public RestIterator( |
| 68 | 71 |
final String baseUrl, |
| 69 | 72 |
final String resumptionType, |
| ... | ... | |
| 134 | 137 |
synchronized (recordQueue) {
|
| 135 | 138 |
while (recordQueue.isEmpty() && !query.isEmpty() ) {
|
| 136 | 139 |
try {
|
| 140 |
log.info("get Query: " + query);
|
|
| 137 | 141 |
query = downloadPage(query); |
| 142 |
log.debug("next query from downloadPage method: " + query);
|
|
| 138 | 143 |
} catch(CollectorServiceException e) {
|
| 144 |
log.debug("CollectorPlugin.next()-Exception: " + e);
|
|
| 139 | 145 |
throw new RuntimeException(e); |
| 140 | 146 |
} |
| 141 | 147 |
} |
| ... | ... | |
| 154 | 160 |
String emptyXml = "<"+wrapName+"></"+wrapName+">"; |
| 155 | 161 |
Node resultNode = null; |
| 156 | 162 |
NodeList nodeList = null; |
| 157 |
int discoverResultSize = 0; |
|
| 163 |
String qUrlArgument = ""; |
|
| 164 |
int urlOldResumptionSize = 0; |
|
| 158 | 165 |
|
| 159 | 166 |
try {
|
| 160 |
resultStream = new URL(query).openStream(); |
|
| 167 |
URL qUrl = new URL(query); |
|
| 168 |
|
|
| 169 |
resultStream = qUrl.openStream(); |
|
| 161 | 170 |
if("json".equals(resultFormatValue.toLowerCase())){
|
| 162 | 171 |
resultJson = IOUtils.toString(resultStream,"UTF-8"); |
| 163 | 172 |
|
| ... | ... | |
| 200 | 209 |
break; |
| 201 | 210 |
|
| 202 | 211 |
case "discover": // length of results unknown |
| 203 |
if( (emptyXml.toLowerCase()).equals(resultXml.toLowerCase()) ) {
|
|
| 212 |
qUrlArgument = qUrl.getQuery(); |
|
| 213 |
String[] arrayQUrlArgument = qUrlArgument.split("&");
|
|
| 214 |
for(String arrayUrlArgStr : arrayQUrlArgument ) {
|
|
| 215 |
if(arrayUrlArgStr.startsWith(resumptionParam)) {
|
|
| 216 |
String[] resumptionKeyValue = arrayUrlArgStr.split("=");
|
|
| 217 |
urlOldResumptionSize = Integer.parseInt(resumptionKeyValue[1]); |
|
| 218 |
log.debug("discover OldResumptionSize from Url: " + urlOldResumptionSize);
|
|
| 219 |
} |
|
| 220 |
} |
|
| 221 |
|
|
| 222 |
// |
|
| 223 |
if( ( (emptyXml.toLowerCase()).equals(resultXml.toLowerCase()) ) |
|
| 224 |
|| ( (nodeList != null) && (nodeList.getLength() < resultSizeValue) ) |
|
| 225 |
) {
|
|
| 204 | 226 |
// resumptionStr = ""; |
| 227 |
if(nodeList != null) { discoverResultSize += nodeList.getLength(); }
|
|
| 205 | 228 |
resultTotal = discoverResultSize; |
| 206 | 229 |
} else {
|
| 207 | 230 |
resumptionStr = Integer.toString(resumptionInt); |
Also available in: Unified diff
Additional to discover option in the Rest_Json CollectorPlugin for the enhancements of the new OpenDOAR API at JISC under https://v2.sherpa.ac.uk/opendoar/