Revision 60307
Added by Alessia Bardi about 3 years ago
| modules/dnet-collector-plugins/trunk/src/main/java/eu/dnetlib/data/collector/plugins/rest/RestCollectorPlugin.java | ||
|---|---|---|
| 32 | 32 |
final String authMethod = ifDescriptor.getParams().get("authMethod");
|
| 33 | 33 |
final String authToken = ifDescriptor.getParams().get("authToken");
|
| 34 | 34 |
|
| 35 |
|
|
| 36 | 35 |
if (StringUtils.isBlank(baseUrl)) {throw new CollectorServiceException("Param 'baseUrl' is null or empty");}
|
| 37 | 36 |
if (StringUtils.isBlank(resumptionType)) {throw new CollectorServiceException("Param 'resumptionType' is null or empty");}
|
| 38 | 37 |
if (StringUtils.isBlank(resumptionParam)) {throw new CollectorServiceException("Param 'resumptionParam' is null or empty");}
|
| 39 | 38 |
// if (StringUtils.isBlank(resumptionXpath)) {throw new CollectorServiceException("Param 'resumptionXpath' is null or empty");}
|
| 40 | 39 |
// if (StringUtils.isBlank(resultTotalXpath)) {throw new CollectorServiceException("Param 'resultTotalXpath' is null or empty");}
|
| 41 | 40 |
// resultFormatParam can be emtpy because some Rest-APIs doesn't like this argument in the query |
| 42 |
//if (StringUtils.isBlank(resultFormatParam)) {throw new CollectorServiceException("Param 'resultFormatParam' is null, empty or whitespace");}
|
|
| 41 |
//if (StringUtils.isBlank(resultFormatParam)) {throw new CollectorServiceException("Param 'resultFormatParam' is null, empty or whitespace");}
|
|
| 43 | 42 |
if (StringUtils.isBlank(resultFormatValue)) {throw new CollectorServiceException("Param 'resultFormatValue' is null or empty");}
|
| 44 |
// if (StringUtils.isBlank(resultSizeParam)) {throw new CollectorServiceException("Param 'resultSizeParam' is null or empty");}
|
|
| 45 |
// prevent resumptionType: discover -- if (Integer.valueOf(resultSizeValue) <= 1) {throw new CollectorServiceException("Param 'resultSizeValue' is less than 2");}
|
|
| 46 |
if (StringUtils.isBlank(queryParams)) {throw new CollectorServiceException("Param 'queryParams' is null or empty");}
|
|
| 43 |
// if (StringUtils.isBlank(resultSizeParam)) {throw new CollectorServiceException("Param 'resultSizeParam' is null or empty");}
|
|
| 44 |
// prevent resumptionType: discover -- if (Integer.valueOf(resultSizeValue) <= 1) {throw new CollectorServiceException("Param 'resultSizeValue' is less than 2");}
|
|
| 45 |
if (StringUtils.isBlank(queryParams)) {throw new CollectorServiceException("Param 'queryParams' is null or empty");}
|
|
| 47 | 46 |
if (StringUtils.isBlank(entityXpath)) {throw new CollectorServiceException("Param 'entityXpath' is null or empty");}
|
| 47 |
|
|
| 48 |
String resFormat = ifDescriptor.getParams().get("resultOutputFormat");
|
|
| 49 |
final String resultOutputFormat = StringUtils.isNotBlank(resFormat) ? resFormat.toLowerCase() : resultFormatValue.toLowerCase(); |
|
| 48 | 50 |
|
| 49 | 51 |
return () -> new RestIterator( |
| 50 | 52 |
baseUrl, |
| ... | ... | |
| 59 | 61 |
queryParams, |
| 60 | 62 |
entityXpath, |
| 61 | 63 |
authMethod, |
| 62 |
authToken); |
|
| 64 |
authToken, |
|
| 65 |
resultOutputFormat); |
|
| 63 | 66 |
} |
| 64 | 67 |
|
| 65 | 68 |
} |
| modules/dnet-collector-plugins/trunk/src/main/java/eu/dnetlib/data/collector/plugins/rest/RestIterator.java | ||
|---|---|---|
| 70 | 70 |
private Queue<String> recordQueue = new PriorityBlockingQueue<String>(); |
| 71 | 71 |
private int discoverResultSize = 0; |
| 72 | 72 |
private int pagination = 1; |
| 73 |
/* |
|
| 74 |
While resultFormatValue is added to the request parameter, this is used to say that the results are retrieved in json. |
|
| 75 |
useful for cases when the target API expects a resultFormatValue != json, but the results are returned in json. |
|
| 76 |
An example is the EU Open Data Portal API: resultFormatValue=standard, results are in json format. |
|
| 77 |
*/ |
|
| 78 |
private String resultOutputFormat; |
|
| 73 | 79 |
|
| 74 | 80 |
|
| 75 |
/** |
|
| 76 |
* RestIterator class |
|
| 77 |
* |
|
| 78 |
* compatible to version before 1.3.33 |
|
| 79 |
* |
|
| 80 |
* @param baseUrl |
|
| 81 |
* @param resumptionType |
|
| 82 |
* @param resumptionParam |
|
| 83 |
* @param resumptionXpath |
|
| 84 |
* @param resultTotalXpath |
|
| 85 |
* @param resultFormatParam |
|
| 86 |
* @param resultFormatValue |
|
| 87 |
* @param resultSizeParam |
|
| 88 |
* @param resultSizeValueStr |
|
| 89 |
* @param queryParams |
|
| 90 |
* @param entityXpath |
|
| 91 |
*/ |
|
| 92 | 81 |
public RestIterator( |
| 93 | 82 |
final String baseUrl, |
| 94 | 83 |
final String resumptionType, |
| ... | ... | |
| 100 | 89 |
final String resultSizeParam, |
| 101 | 90 |
final String resultSizeValueStr, |
| 102 | 91 |
final String queryParams, |
| 103 |
final String entityXpath |
|
| 104 |
) {
|
|
| 105 |
this(baseUrl,resumptionType,resumptionParam,resumptionXpath,resultTotalXpath,resultFormatParam,resultFormatValue,resultSizeParam,resultSizeValueStr,queryParams,entityXpath,"", ""); |
|
| 106 |
} |
|
| 107 |
|
|
| 108 |
public RestIterator( |
|
| 109 |
final String baseUrl, |
|
| 110 |
final String resumptionType, |
|
| 111 |
final String resumptionParam, |
|
| 112 |
final String resumptionXpath, |
|
| 113 |
final String resultTotalXpath, |
|
| 114 |
final String resultFormatParam, |
|
| 115 |
final String resultFormatValue, |
|
| 116 |
final String resultSizeParam, |
|
| 117 |
final String resultSizeValueStr, |
|
| 118 |
final String queryParams, |
|
| 119 | 92 |
final String entityXpath, |
| 120 | 93 |
final String authMethod, |
| 121 | 94 |
final String authToken, |
| 122 |
final String resultOffsetParam
|
|
| 95 |
final String resultOutputFormat
|
|
| 123 | 96 |
) {
|
| 124 |
this(baseUrl,resumptionType,resumptionParam,resumptionXpath,resultTotalXpath,resultFormatParam,resultFormatValue,resultSizeParam,resultSizeValueStr,queryParams,entityXpath,"", ""); |
|
| 125 |
} |
|
| 126 |
|
|
| 127 |
/** RestIterator class |
|
| 128 |
* compatible to version 1.3.33 |
|
| 129 |
* @param baseUrl |
|
| 130 |
* @param resumptionType |
|
| 131 |
* @param resumptionParam |
|
| 132 |
* @param resumptionXpath |
|
| 133 |
* @param resultTotalXpath |
|
| 134 |
* @param resultFormatParam |
|
| 135 |
* @param resultFormatValue |
|
| 136 |
* @param resultSizeParam |
|
| 137 |
* @param resultSizeValueStr |
|
| 138 |
* @param queryParams |
|
| 139 |
* @param entityXpath |
|
| 140 |
* @param authMethod |
|
| 141 |
* @param authToken |
|
| 142 |
*/ |
|
| 143 |
public RestIterator( |
|
| 144 |
final String baseUrl, |
|
| 145 |
final String resumptionType, |
|
| 146 |
final String resumptionParam, |
|
| 147 |
final String resumptionXpath, |
|
| 148 |
final String resultTotalXpath, |
|
| 149 |
final String resultFormatParam, |
|
| 150 |
final String resultFormatValue, |
|
| 151 |
final String resultSizeParam, |
|
| 152 |
final String resultSizeValueStr, |
|
| 153 |
final String queryParams, |
|
| 154 |
final String entityXpath, |
|
| 155 |
final String authMethod, |
|
| 156 |
final String authToken |
|
| 157 |
) {
|
|
| 158 | 97 |
this.jsonUtils = new JsonUtils(); |
| 159 | 98 |
this.baseUrl = baseUrl; |
| 160 | 99 |
this.resumptionType = resumptionType; |
| ... | ... | |
| 164 | 103 |
this.resultSizeValue = Integer.valueOf(resultSizeValueStr); |
| 165 | 104 |
this.authMethod = authMethod; |
| 166 | 105 |
this.authToken = authToken; |
| 106 |
this.resultOutputFormat = resultOutputFormat; |
|
| 167 | 107 |
|
| 168 | 108 |
queryFormat = StringUtils.isNotBlank(resultFormatParam) ? "&" + resultFormatParam + "=" + resultFormatValue : ""; |
| 169 | 109 |
querySize = StringUtils.isNotBlank(resultSizeParam) ? "&" + resultSizeParam + "=" + resultSizeValueStr : ""; |
| ... | ... | |
| 273 | 213 |
} |
| 274 | 214 |
|
| 275 | 215 |
resultStream = theHttpInputStream; |
| 276 |
if ("json".equals(resultFormatValue.toLowerCase())) {
|
|
| 216 |
if ("json".equals(resultOutputFormat)) {
|
|
| 277 | 217 |
resultJson = IOUtils.toString(resultStream, "UTF-8"); |
| 278 | 218 |
resultXml = jsonUtils.convertToXML(resultJson); |
| 279 | 219 |
resultStream = IOUtils.toInputStream(resultXml, "UTF-8"); |
| ... | ... | |
| 423 | 363 |
} |
| 424 | 364 |
} |
| 425 | 365 |
|
| 366 |
public String getResultFormatValue() {
|
|
| 367 |
return resultFormatValue; |
|
| 368 |
} |
|
| 369 |
|
|
| 370 |
public String getResultOutputFormat() {
|
|
| 371 |
return resultOutputFormat; |
|
| 372 |
} |
|
| 373 |
|
|
| 426 | 374 |
} |
| modules/dnet-collector-plugins/trunk/src/main/resources/eu/dnetlib/data/collector/plugins/applicationContext-dnet-modular-collector-plugins.xml | ||
|---|---|---|
| 196 | 196 |
p:name="authToken" p:optional="true" /> |
| 197 | 197 |
<bean class="eu.dnetlib.data.collector.rmi.ProtocolParameter" |
| 198 | 198 |
p:name="entityXpath" /> |
| 199 |
<bean class="eu.dnetlib.data.collector.rmi.ProtocolParameter" |
|
| 200 |
p:name="resultOutputFormat" p:optional="true" /> |
|
| 199 | 201 |
</list> |
| 200 | 202 |
</property> |
| 201 | 203 |
</bean> |
Also available in: Unified diff
Added new optional parameter in case the resultFormatValue is not 'json' but the output is in json format