Project

General

Profile

« Previous | Next » 

Revision 58456

Added by Andreas Czerniak almost 4 years ago

Rest-Collector plugin: changes deep-cursor for CrossRef, v1.3.39-SNAPSHOT

View differences:

modules/dnet-collector-plugins/trunk/src/main/java/eu/dnetlib/data/collector/plugins/rest/RestCollectorPlugin.java
59 59
				queryParams,
60 60
				entityXpath,
61 61
				authMethod,
62
				authToken,
63
				resultOffsetParam);
62
				authToken);
64 63
	}
65 64

  
66 65
}
modules/dnet-collector-plugins/trunk/src/main/java/eu/dnetlib/data/collector/plugins/rest/RestIterator.java
7 7

  
8 8
import java.io.InputStream;
9 9
import java.io.StringWriter;
10
import java.io.UnsupportedEncodingException;
10 11
import java.net.URL;
12
import java.net.URLEncoder;
13
import java.nio.charset.StandardCharsets;
11 14
import java.net.HttpURLConnection;
12 15
import java.util.Iterator;
13 16
import java.util.Queue;
......
32 35

  
33 36
/**
34 37
 * @author Jochen Schirrwagen, Aenne Loehden, Andreas Czerniak
35
 * @date 2020-03-20
38
 * @date 2020-04-09
36 39
 *
37 40
 */
38 41
public class RestIterator implements Iterator<String> {
......
64 67
	private Queue<String> recordQueue = new PriorityBlockingQueue<String>();
65 68
	private int discoverResultSize = 0;
66 69
	private int pagination = 1;
67
	private String resultOffsetParam;
68 70

  
69 71
	/**
70 72
	 * RestIterator class
......
96 98
			final String queryParams,
97 99
			final String entityXpath
98 100
	) {
99
		this(baseUrl,resumptionType,resumptionParam,resumptionXpath,resultTotalXpath,resultFormatParam,resultFormatValue,resultSizeParam,resultSizeValueStr,queryParams,entityXpath,"", "","");
101
		this(baseUrl,resumptionType,resumptionParam,resumptionXpath,resultTotalXpath,resultFormatParam,resultFormatValue,resultSizeParam,resultSizeValueStr,queryParams,entityXpath,"", "");
100 102
	}
101 103

  
102 104
	/** RestIterator class
......
130 132
			final String authMethod,
131 133
			final String authToken
132 134
	) {
133
		this(baseUrl,resumptionType,resumptionParam,resumptionXpath,resultTotalXpath,resultFormatParam,resultFormatValue,resultSizeParam,resultSizeValueStr,queryParams,entityXpath,"", "","");
134
	}
135
	
136
	/**
137
	 * 
138
	 * @param baseUrl
139
	 * @param resumptionType
140
	 * @param resumptionParam
141
	 * @param resumptionXpath
142
	 * @param resultTotalXpath
143
	 * @param resultFormatParam
144
	 * @param resultFormatValue
145
	 * @param resultSizeParam
146
	 * @param resultSizeValueStr
147
	 * @param queryParams
148
	 * @param entityXpath
149
	 * @param authMethod
150
	 * @param authToken
151
	 * @param resultOffsetParam
152
	 */
153
	public RestIterator(
154
			final String baseUrl,
155
			final String resumptionType,
156
			final String resumptionParam,
157
			final String resumptionXpath,
158
			final String resultTotalXpath,
159
			final String resultFormatParam,
160
			final String resultFormatValue,
161
			final String resultSizeParam,
162
			final String resultSizeValueStr,
163
			final String queryParams,
164
			final String entityXpath,
165
			final String authMethod,
166
			final String authToken,
167
			final String resultOffsetParam
168
	) {
169 135
		this.baseUrl = baseUrl;
170 136
		this.resumptionType = resumptionType;
171 137
		this.resumptionParam = resumptionParam;
......
174 140
		this.resultSizeValue = Integer.valueOf(resultSizeValueStr);
175 141
		this.authMethod = authMethod;
176 142
		this.authToken = authToken;
177
		this.resultOffsetParam = resultOffsetParam;
178 143

  
179 144
		queryFormat = StringUtils.isNotBlank(resultFormatParam) ? "&" + resultFormatParam + "=" + resultFormatValue : "";
180 145
		querySize = StringUtils.isNotBlank(resultSizeParam) ? "&" + resultSizeParam + "=" + resultSizeValueStr : "";
......
348 313

  
349 314
			case "deep-cursor":   // size of result items unknown, iterate over items  (for supporting deep cursor in solr)
350 315
				// isn't relevant -- if (resultSizeValue < 2) {throw new CollectorServiceException("Mode: deep-cursor, Param 'resultSizeValue' is less than 2");}
351
				// if ( (this.resultOffsetParam.length()) < 2) {throw new CollectorServiceException("Mode: deep-cursor, Param 'resultOffsetParam' is less than 2");}
352 316

  
353
				resumptionStr = xprResumptionPath.evaluate(resultNode);
317
				resumptionStr = encodeValue(xprResumptionPath.evaluate(resultNode));
354 318
				queryParams = queryParams.replace("&cursor=*", "");
355
				// queryParams = queryParams.replace("&&",  "&");
356 319
				
357
				//resumptionStr += "&" + resultOffsetParam + "=" + Integer.toString(resumptionInt);
358
				
320
				resumptionInt += nodeList.getLength();
359 321
				log.debug("downloadPage().deep-cursor: resumptionStr=" + resumptionStr + " ; queryParams=" + queryParams);
360 322

  
361 323
				break;
......
490 452

  
491 453
		return isValidInteger;
492 454
	}
455
	
456
	// Method to encode a string value using `UTF-8` encoding scheme
457
    private String encodeValue(String value) {
458
        try {
459
            return URLEncoder.encode(value, StandardCharsets.UTF_8.toString());
460
        } catch (UnsupportedEncodingException ex) {
461
            throw new RuntimeException(ex.getCause());
462
        }
463
    }
493 464

  
494 465
}
modules/dnet-collector-plugins/trunk/src/main/resources/eu/dnetlib/data/collector/plugins/applicationContext-dnet-modular-collector-plugins.xml
194 194
							p:name="authMethod" p:optional="true" />	
195 195
						<bean class="eu.dnetlib.data.collector.rmi.ProtocolParameter"
196 196
							p:name="authToken" p:optional="true" />	
197
                        <bean class="eu.dnetlib.data.collector.rmi.ProtocolParameter"
198
							p:name="resultOffsetParam" p:optional="true" />								
199 197
						<bean class="eu.dnetlib.data.collector.rmi.ProtocolParameter"
200 198
							p:name="entityXpath" />
201 199
					</list>
modules/dnet-collector-plugins/trunk/pom.xml
7 7
	</parent>
8 8
	<groupId>eu.dnetlib</groupId>
9 9
	<artifactId>dnet-collector-plugins</artifactId>
10
	<version>1.3.38-SNAPSHOT</version>
10
	<version>1.3.39-SNAPSHOT</version>
11 11
	<scm>
12 12
		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-collector-plugins/trunk</developerConnection>
13 13
	</scm>

Also available in: Unified diff