Project

General

Profile

« Previous | Next » 

Revision 57207

Reports always download the first 2000 results. Dedup organizations returned #4588

View differences:

modules/uoa-search/trunk/src/main/java/eu/dnetlib/data/search/solr/SolrResultSet.java
177 177

  
178 178
        queryOpts.add("start", from*to + "");
179 179
        queryOpts.add("rows", to +"");
180
        //queryOpts.add("f.resulthostingdatasource.facet.limit", "2");
180 181

  
181 182
        long startTime = System.nanoTime();
182 183

  
......
294 295
        return null;
295 296
    }
296 297

  
297
    public void cursorGet(Transformer transformer, OutputStream os) throws SolrServerException, SearchServiceException {
298
    /**
299
     * limit is the maximum number of results the cursor get is allowed to fetch. If limit is set to -1 all
300
     * results are returned.
301
     */
302
    public void cursorGet(Transformer transformer, int limit, OutputStream os) throws SolrServerException, SearchServiceException {
303

  
304
        int rows = 500;
305
        int limitCounter = -1;
306

  
298 307
        queryOpts.add("start", "0");
299 308
        queryOpts.add("rows", "0");
300 309
        queryOpts.remove("rows");
301
        queryOpts.add("rows", "500");
310
        queryOpts.add("rows", rows+"");
302 311
        queryOpts.add("fl", "__result");
303 312
        queryOpts.add("shards.tolerant","true");
304 313
        queryOpts.add("cursorMark", "*");
......
307 316
        String cursorMark = "*";
308 317
        String nextCursorMark = "";
309 318

  
310
        int curs = 0;
319
        if ( limit > 0 ) {
320
            limitCounter = limit/rows;
321
            logger.info("limit counter " + limitCounter);
322
        }
323

  
311 324
        try {
312 325
            QueryResponse resp = solrClient.query(SolrParams.toSolrParams(queryOpts));
313 326

  
314
            while (!cursorMark.equals(nextCursorMark)) {
327
            while (!cursorMark.equals(nextCursorMark) && ( limitCounter > 0 || limitCounter == -1)) {
328
                //logger.info(">> " + limitCounter);
315 329
                resp = solrClient.query(SolrParams.toSolrParams(queryOpts));
316 330
                cursorMark = nextCursorMark;
317 331
                nextCursorMark = resp.getNextCursorMark();
......
320 334
                    if (transformer != null) {
321 335
                        String result = null;
322 336
                        try {
337
                            logger.debug("PRE RESULT " + resp.getResults().get(i).get("__result"));
323 338
                            result = transformer.transform((String) resp.getResults().get(i).get("__result"));
324 339
                            logger.debug("RESULT " + result);
325 340

  
......
340 355

  
341 356
                queryOpts.remove("cursorMark");
342 357
                queryOpts.add("cursorMark", nextCursorMark);
343
                curs++;
358
                limitCounter--;
344 359
            }
345 360

  
346 361
        } catch (IOException ioe) {
347 362
            logger.error("Error executing solr query. ", ioe);
348 363
        }
349

  
350
        logger.debug("CURS " + curs);
351 364
    }
352 365

  
353 366

  
modules/uoa-search/trunk/src/main/java/eu/dnetlib/data/search/app/SearchServiceImpl.java
313 313

  
314 314
            rs = rsFactory.createResultSet(epr);
315 315

  
316
            ((SolrResultSet)rs).cursorGet(transformer,os);
316
            ((SolrResultSet)rs).cursorGet(transformer,2000, os);
317 317

  
318 318
        } catch (IndexServiceException ise) {
319 319
            logger.error("Error getting cursor results.", ise);
modules/uoa-search/trunk/src/main/java/eu/dnetlib/data/search/web/api/SearchApiService.java
705 705
        try {
706 706
            SearchResult sr = ((SearchServiceImpl)searchService).newSearch(fullQuery, Locale.getDefault().toString(), fields, fieldQueries,0, 0, format, transformer,null,false);
707 707
            logger.debug("Total number of results " + sr.getTotal());
708
            
709
            if (sr.getTotal() > 10000) {
710
                return Response.status(Response.Status.BAD_REQUEST).entity(APIResponseFormatter.
711
                        compose500Message(MediaType.APPLICATION_JSON, "Fail to fetch report.", "You have exceeded the number of allowed returned results")).build();
712
            }
713 708

  
709

  
714 710
        } catch (SearchServiceException sse) {
715 711
            return Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(APIResponseFormatter.
716 712
                    compose500Message(MediaType.APPLICATION_JSON, "Fail to fetch report.", sse.getMessage())).build();
......
851 847
        enhanceQueryWithEntityId(queryBuilder, entity, cleanId(entityId));
852 848
    }
853 849

  
854
    /* Queries with pids are only valid with deleted by inderence marked as true */
850
    /* Queries with pids are only valid with deleted by inference marked as true */
855 851
    private void resultPidQuery(StringBuilder queryBuilder, RequestResponseHandler.Entity entity, String resultPid, List<String> fieldQueries){
856 852
        builtDeletedByInferenceQuery(queryBuilder);
857
        if (entity == RequestResponseHandler.Entity.ORGANIZATION) {
858
            enhanceFieldQueryWithEntityType(entity, fieldQueries);
859
        } else if (entity != RequestResponseHandler.Entity.NONE) {
853

  
854
        if (entity != RequestResponseHandler.Entity.NONE && entity != RequestResponseHandler.Entity.ORGANIZATION) {
860 855
            throw new IllegalArgumentException("Entity " + entity.toString() + " is not supported.");
861 856
        }
862 857
        enhanceQueryWithPid(queryBuilder, entity, cleanId(resultPid));
modules/uoa-search/trunk/src/main/java/eu/dnetlib/data/search/web/api/SearchRequestController.java
6 6
import eu.dnetlib.data.search.utils.cql.ParameterQueryEnhancer;
7 7
import eu.dnetlib.data.search.utils.vocabulary.VocabularyManager;
8 8
import eu.dnetlib.domain.data.FormattedSearchResult;
9
import eu.dnetlib.functionality.cql.CqlTranslatorImpl;
9 10
import io.micrometer.core.annotation.Timed;
10 11
import io.micrometer.prometheus.PrometheusMeterRegistry;
11 12
import org.apache.commons.collections.ListUtils;
12 13
import org.apache.commons.lang.IncompleteArgumentException;
13 14
import org.apache.commons.lang.StringEscapeUtils;
14 15
import org.apache.log4j.Logger;
16
import org.apache.solr.client.solrj.SolrServerException;
17
import org.apache.solr.client.solrj.impl.CloudSolrClient;
18
import org.apache.solr.client.solrj.response.QueryResponse;
19
import org.apache.solr.common.params.SolrParams;
20
import org.apache.solr.common.util.NamedList;
15 21
import org.springframework.beans.factory.annotation.Autowired;
16 22
import org.springframework.context.annotation.EnableAspectJAutoProxy;
17 23
import org.springframework.http.HttpStatus;
......
23 29
import org.springframework.web.bind.annotation.RequestMapping;
24 30
import org.springframework.web.bind.annotation.RequestMethod;
25 31
import org.springframework.web.bind.annotation.ResponseBody;
32
import org.z3950.zing.cql.CQLParseException;
26 33

  
27 34
import javax.annotation.Resource;
35
import javax.servlet.ServletOutputStream;
28 36
import javax.servlet.http.HttpServletRequest;
29 37
import javax.servlet.http.HttpServletResponse;
38
import java.io.IOException;
30 39
import java.io.PrintWriter;
31 40
import java.text.SimpleDateFormat;
32 41
import java.util.*;
......
739 748
        return request.getParameter("format");
740 749
    } */
741 750

  
742
    /*public void test(HttpServletRequest request, HttpServletResponse response) throws IOException, SolrServerException {
751
    /*public static void main() throws IOException, SolrServerException {
743 752

  
744 753
        long time = System.currentTimeMillis();
745 754

  
746
        CloudSolrServer solrClient = new CloudSolrServer("openaire-solr-beta.vls.icm.edu.pl:9983");
755
        CloudSolrClient solrClient = new CloudSolrClient.Builder().
756
                withZkHost(Arrays.asList(new String[]{"quorum0.t.hadoop.research-infrastructures.eu:2182",
757
                        "quorum1.t.hadoop.research-infrastructures.eu:2182",
758
                        "quorum2.t.hadoop.research-infrastructures.eu:2182",
759
                        "quorum3.t.hadoop.research-infrastructures.eu:2182",
760
                        "quorum4.t.hadoop.research-infrastructures.eu:2182/solr-dev-openaire"})).build();
747 761
        solrClient.setDefaultCollection("DMF-index-openaire");
748 762

  
749
        response.setContentType("text/html");
750
        ServletOutputStream out=response.getOutputStream();
763
       // response.setContentType("text/html");
764
       // ServletOutputStream out=response.getOutputStream();
751 765

  
752 766
        NamedList<String> queryOpts = new NamedList<String>();
753 767

  
......
792 806

  
793 807
            for (int i = 0; i < resp.getResults().size(); i++) {
794 808
                String result = ((ArrayList<String>) resp.getResults().get(i).get("__result")).get(0);
795
                out.write(result.getBytes());
796
                out.flush();
809
             //   out.write(result.getBytes());
810
             //   out.flush();
797 811
            }
798 812

  
799 813
            System.out.println("END");
......
807 821

  
808 822
        }
809 823

  
810
        out.close();
824
      //  out.close();
811 825

  
812 826
        time = System.currentTimeMillis() - time;
813 827
        System.out.println("Answer time " + time);

Also available in: Unified diff