Revision 57207
Added by Katerina Iatropoulou almost 4 years ago
modules/uoa-search/trunk/src/main/java/eu/dnetlib/data/search/solr/SolrResultSet.java | ||
---|---|---|
177 | 177 |
|
178 | 178 |
queryOpts.add("start", from*to + ""); |
179 | 179 |
queryOpts.add("rows", to +""); |
180 |
//queryOpts.add("f.resulthostingdatasource.facet.limit", "2"); |
|
180 | 181 |
|
181 | 182 |
long startTime = System.nanoTime(); |
182 | 183 |
|
... | ... | |
294 | 295 |
return null; |
295 | 296 |
} |
296 | 297 |
|
297 |
public void cursorGet(Transformer transformer, OutputStream os) throws SolrServerException, SearchServiceException { |
|
298 |
/** |
|
299 |
* limit is the maximum number of results the cursor get is allowed to fetch. If limit is set to -1 all |
|
300 |
* results are returned. |
|
301 |
*/ |
|
302 |
public void cursorGet(Transformer transformer, int limit, OutputStream os) throws SolrServerException, SearchServiceException { |
|
303 |
|
|
304 |
int rows = 500; |
|
305 |
int limitCounter = -1; |
|
306 |
|
|
298 | 307 |
queryOpts.add("start", "0"); |
299 | 308 |
queryOpts.add("rows", "0"); |
300 | 309 |
queryOpts.remove("rows"); |
301 |
queryOpts.add("rows", "500");
|
|
310 |
queryOpts.add("rows", rows+"");
|
|
302 | 311 |
queryOpts.add("fl", "__result"); |
303 | 312 |
queryOpts.add("shards.tolerant","true"); |
304 | 313 |
queryOpts.add("cursorMark", "*"); |
... | ... | |
307 | 316 |
String cursorMark = "*"; |
308 | 317 |
String nextCursorMark = ""; |
309 | 318 |
|
310 |
int curs = 0; |
|
319 |
if ( limit > 0 ) { |
|
320 |
limitCounter = limit/rows; |
|
321 |
logger.info("limit counter " + limitCounter); |
|
322 |
} |
|
323 |
|
|
311 | 324 |
try { |
312 | 325 |
QueryResponse resp = solrClient.query(SolrParams.toSolrParams(queryOpts)); |
313 | 326 |
|
314 |
while (!cursorMark.equals(nextCursorMark)) { |
|
327 |
while (!cursorMark.equals(nextCursorMark) && ( limitCounter > 0 || limitCounter == -1)) { |
|
328 |
//logger.info(">> " + limitCounter); |
|
315 | 329 |
resp = solrClient.query(SolrParams.toSolrParams(queryOpts)); |
316 | 330 |
cursorMark = nextCursorMark; |
317 | 331 |
nextCursorMark = resp.getNextCursorMark(); |
... | ... | |
320 | 334 |
if (transformer != null) { |
321 | 335 |
String result = null; |
322 | 336 |
try { |
337 |
logger.debug("PRE RESULT " + resp.getResults().get(i).get("__result")); |
|
323 | 338 |
result = transformer.transform((String) resp.getResults().get(i).get("__result")); |
324 | 339 |
logger.debug("RESULT " + result); |
325 | 340 |
|
... | ... | |
340 | 355 |
|
341 | 356 |
queryOpts.remove("cursorMark"); |
342 | 357 |
queryOpts.add("cursorMark", nextCursorMark); |
343 |
curs++;
|
|
358 |
limitCounter--;
|
|
344 | 359 |
} |
345 | 360 |
|
346 | 361 |
} catch (IOException ioe) { |
347 | 362 |
logger.error("Error executing solr query. ", ioe); |
348 | 363 |
} |
349 |
|
|
350 |
logger.debug("CURS " + curs); |
|
351 | 364 |
} |
352 | 365 |
|
353 | 366 |
|
modules/uoa-search/trunk/src/main/java/eu/dnetlib/data/search/app/SearchServiceImpl.java | ||
---|---|---|
313 | 313 |
|
314 | 314 |
rs = rsFactory.createResultSet(epr); |
315 | 315 |
|
316 |
((SolrResultSet)rs).cursorGet(transformer,os); |
|
316 |
((SolrResultSet)rs).cursorGet(transformer,2000, os);
|
|
317 | 317 |
|
318 | 318 |
} catch (IndexServiceException ise) { |
319 | 319 |
logger.error("Error getting cursor results.", ise); |
modules/uoa-search/trunk/src/main/java/eu/dnetlib/data/search/web/api/SearchApiService.java | ||
---|---|---|
705 | 705 |
try { |
706 | 706 |
SearchResult sr = ((SearchServiceImpl)searchService).newSearch(fullQuery, Locale.getDefault().toString(), fields, fieldQueries,0, 0, format, transformer,null,false); |
707 | 707 |
logger.debug("Total number of results " + sr.getTotal()); |
708 |
|
|
709 |
if (sr.getTotal() > 10000) { |
|
710 |
return Response.status(Response.Status.BAD_REQUEST).entity(APIResponseFormatter. |
|
711 |
compose500Message(MediaType.APPLICATION_JSON, "Fail to fetch report.", "You have exceeded the number of allowed returned results")).build(); |
|
712 |
} |
|
713 | 708 |
|
709 |
|
|
714 | 710 |
} catch (SearchServiceException sse) { |
715 | 711 |
return Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(APIResponseFormatter. |
716 | 712 |
compose500Message(MediaType.APPLICATION_JSON, "Fail to fetch report.", sse.getMessage())).build(); |
... | ... | |
851 | 847 |
enhanceQueryWithEntityId(queryBuilder, entity, cleanId(entityId)); |
852 | 848 |
} |
853 | 849 |
|
854 |
/* Queries with pids are only valid with deleted by inderence marked as true */
|
|
850 |
/* Queries with pids are only valid with deleted by inference marked as true */
|
|
855 | 851 |
private void resultPidQuery(StringBuilder queryBuilder, RequestResponseHandler.Entity entity, String resultPid, List<String> fieldQueries){ |
856 | 852 |
builtDeletedByInferenceQuery(queryBuilder); |
857 |
if (entity == RequestResponseHandler.Entity.ORGANIZATION) { |
|
858 |
enhanceFieldQueryWithEntityType(entity, fieldQueries); |
|
859 |
} else if (entity != RequestResponseHandler.Entity.NONE) { |
|
853 |
|
|
854 |
if (entity != RequestResponseHandler.Entity.NONE && entity != RequestResponseHandler.Entity.ORGANIZATION) { |
|
860 | 855 |
throw new IllegalArgumentException("Entity " + entity.toString() + " is not supported."); |
861 | 856 |
} |
862 | 857 |
enhanceQueryWithPid(queryBuilder, entity, cleanId(resultPid)); |
modules/uoa-search/trunk/src/main/java/eu/dnetlib/data/search/web/api/SearchRequestController.java | ||
---|---|---|
6 | 6 |
import eu.dnetlib.data.search.utils.cql.ParameterQueryEnhancer; |
7 | 7 |
import eu.dnetlib.data.search.utils.vocabulary.VocabularyManager; |
8 | 8 |
import eu.dnetlib.domain.data.FormattedSearchResult; |
9 |
import eu.dnetlib.functionality.cql.CqlTranslatorImpl; |
|
9 | 10 |
import io.micrometer.core.annotation.Timed; |
10 | 11 |
import io.micrometer.prometheus.PrometheusMeterRegistry; |
11 | 12 |
import org.apache.commons.collections.ListUtils; |
12 | 13 |
import org.apache.commons.lang.IncompleteArgumentException; |
13 | 14 |
import org.apache.commons.lang.StringEscapeUtils; |
14 | 15 |
import org.apache.log4j.Logger; |
16 |
import org.apache.solr.client.solrj.SolrServerException; |
|
17 |
import org.apache.solr.client.solrj.impl.CloudSolrClient; |
|
18 |
import org.apache.solr.client.solrj.response.QueryResponse; |
|
19 |
import org.apache.solr.common.params.SolrParams; |
|
20 |
import org.apache.solr.common.util.NamedList; |
|
15 | 21 |
import org.springframework.beans.factory.annotation.Autowired; |
16 | 22 |
import org.springframework.context.annotation.EnableAspectJAutoProxy; |
17 | 23 |
import org.springframework.http.HttpStatus; |
... | ... | |
23 | 29 |
import org.springframework.web.bind.annotation.RequestMapping; |
24 | 30 |
import org.springframework.web.bind.annotation.RequestMethod; |
25 | 31 |
import org.springframework.web.bind.annotation.ResponseBody; |
32 |
import org.z3950.zing.cql.CQLParseException; |
|
26 | 33 |
|
27 | 34 |
import javax.annotation.Resource; |
35 |
import javax.servlet.ServletOutputStream; |
|
28 | 36 |
import javax.servlet.http.HttpServletRequest; |
29 | 37 |
import javax.servlet.http.HttpServletResponse; |
38 |
import java.io.IOException; |
|
30 | 39 |
import java.io.PrintWriter; |
31 | 40 |
import java.text.SimpleDateFormat; |
32 | 41 |
import java.util.*; |
... | ... | |
739 | 748 |
return request.getParameter("format"); |
740 | 749 |
} */ |
741 | 750 |
|
742 |
/*public void test(HttpServletRequest request, HttpServletResponse response) throws IOException, SolrServerException {
|
|
751 |
/*public static void main() throws IOException, SolrServerException {
|
|
743 | 752 |
|
744 | 753 |
long time = System.currentTimeMillis(); |
745 | 754 |
|
746 |
CloudSolrServer solrClient = new CloudSolrServer("openaire-solr-beta.vls.icm.edu.pl:9983"); |
|
755 |
CloudSolrClient solrClient = new CloudSolrClient.Builder(). |
|
756 |
withZkHost(Arrays.asList(new String[]{"quorum0.t.hadoop.research-infrastructures.eu:2182", |
|
757 |
"quorum1.t.hadoop.research-infrastructures.eu:2182", |
|
758 |
"quorum2.t.hadoop.research-infrastructures.eu:2182", |
|
759 |
"quorum3.t.hadoop.research-infrastructures.eu:2182", |
|
760 |
"quorum4.t.hadoop.research-infrastructures.eu:2182/solr-dev-openaire"})).build(); |
|
747 | 761 |
solrClient.setDefaultCollection("DMF-index-openaire"); |
748 | 762 |
|
749 |
response.setContentType("text/html"); |
|
750 |
ServletOutputStream out=response.getOutputStream(); |
|
763 |
// response.setContentType("text/html");
|
|
764 |
// ServletOutputStream out=response.getOutputStream();
|
|
751 | 765 |
|
752 | 766 |
NamedList<String> queryOpts = new NamedList<String>(); |
753 | 767 |
|
... | ... | |
792 | 806 |
|
793 | 807 |
for (int i = 0; i < resp.getResults().size(); i++) { |
794 | 808 |
String result = ((ArrayList<String>) resp.getResults().get(i).get("__result")).get(0); |
795 |
out.write(result.getBytes()); |
|
796 |
out.flush(); |
|
809 |
// out.write(result.getBytes());
|
|
810 |
// out.flush();
|
|
797 | 811 |
} |
798 | 812 |
|
799 | 813 |
System.out.println("END"); |
... | ... | |
807 | 821 |
|
808 | 822 |
} |
809 | 823 |
|
810 |
out.close(); |
|
824 |
// out.close();
|
|
811 | 825 |
|
812 | 826 |
time = System.currentTimeMillis() - time; |
813 | 827 |
System.out.println("Answer time " + time); |
Also available in: Unified diff
Reports always download the first 2000 results. Dedup organizations returned #4588