D-Net

import eu.dnetlib.functionality.cql.CqlTranslatorImpl;

import java.io.OutputStream;

    @Override

    public EPR getEpr() {

        return null;

    }

    public void cursorGet(Transformer transformer, OutputStream os) throws SolrServerException, IOException {

        queryOpts.add("start", "0");

        queryOpts.add("rows", "500");

        queryOpts.add("fl", "__result");

        queryOpts.add("shards.tolerant","true");

        queryOpts.add("cursorMark", "*");

        queryOpts.add("sort", "__indexrecordidentifier asc");

        String cursorMark = "*";

        String nextCursorMark = "";

        QueryResponse resp = solrClient.query(SolrParams.toSolrParams(queryOpts));

        int curs = 0;

        while (!cursorMark.equals(nextCursorMark)) {

            logger.debug("QUERY OPTS: " + queryOpts.get("cursorMark"));

            resp = solrClient.query(SolrParams.toSolrParams(queryOpts));

            logger.debug("TOTAL number " + resp.getResults().getNumFound());

            logger.debug(resp.getNextCursorMark());

            System.out.println("BEGIN");

            System.out.println("cursor " + cursorMark);

            System.out.println("next cursor " + nextCursorMark);

            cursorMark = nextCursorMark;

            nextCursorMark = resp.getNextCursorMark();

            for (int i = 0; i < resp.getResults().size(); i++) {

                if (transformer != null) {

                    String result = null;

                    try {

                        logger.debug(transformer.getClass());

                        result = transformer.transform(((ArrayList<String>) resp.getResults().get(i).get("__result")).get(0));

                        logger.debug("RESULT " + result);

                    } catch (TransformerException e) {

                        logger.error("Something went wrong.", e);

                    }

                    os.write(result.getBytes());

                    os.flush();

                }

            }

            System.out.println("END");

            System.out.println("cursor " + cursorMark);

            System.out.println("next cursor " + nextCursorMark);

            queryOpts.remove("cursorMark");

            queryOpts.add("cursorMark", nextCursorMark);

            System.out.println("CURS " + curs);

            curs ++;

        }

    }

        solrClient.setDefaultCollection("DMF-index-openaire");

        //q=*:*&start=0&rows=10&cursorMark=*&sort=dateofcollection asc

        queryOpts.add("q", new CqlTranslatorImpl().getTranslatedQuery("objIdentifier = acm_________::0002c24f82c295e925a2bdf7bbf49bfc").asLucene());

        queryOpts.add("start", "0");

        queryOpts.add("rows", "1");

        queryOpts.add("fl", "__result");

        queryOpts.add("shards.tolerant","true");

        queryOpts.add("cursorMark", "*");

        queryOpts.add("sort", "__indexrecordidentifier asc");

        //queryOpts.add("q", new CqlTranslatorImpl().getTranslatedQuery("oaftype exact project").asLucene());

        QueryResponse resp = solrClient.query(SolrParams.toSolrParams(queryOpts));

        System.out.println("results " + resp.getResults().size());

        String cursorMark = "*";

        String nextCursorMark = "";

        int curs = 0;

        while (!cursorMark.equals(nextCursorMark)) {

            System.out.println("cursor " + cursorMark);

            System.out.println("next cursor " + nextCursorMark);

            cursorMark = nextCursorMark;

            for (int i = 0; i < resp.getResults().size(); i++) {

                String result = ((ArrayList<String>) resp.getResults().get(i).get("__result")).get(0);

                //System.out.println(result);

                resp = solrClient.query(SolrParams.toSolrParams(queryOpts));

            }

            nextCursorMark = resp.getNextCursorMark();

            queryOpts.add("cursorMark", nextCursorMark);

            System.out.println("CURS " + curs);

            curs ++;

        }

        //System.out.println((new CqlTranslatorImpl().getTranslatedQuery("objIdentifier = acm_________::0002c24f82c295e925a2bdf7bbf49bfc").asLucene()));

        //queryOpts.add("facet", "true");

        //queryOpts.add("facet", "true");

        //queryOpts.add("facet.mincount", "1");

        //queryOpts.add("fq", "popularity");

    }

//    @Override

//    public EPR getEpr() {

//        return epr;

//   }

import org.apache.solr.client.solrj.SolrServerException;

import java.io.IOException;

import java.io.OutputStream;

    public void cursorSearch(String text, List<String> refinefields, List<String> fieldQueries,

                                       String format, Transformer transformer, OutputStream os) throws SearchServiceException {

        long startTime = System.nanoTime();

        IndexService index = getIndexLocator().getService();

        EPR epr = null;

        ResultSet<String> rs = null;

        List<String> browseResults = null;

        List<String> searchResults = null;

        String query = rewrite(text);

        enhanceFieldQueries(fieldQueries);

        logger.info("Performing cursor query " + query + "' and fields " + fieldQueries + " and refine " + refinefields);

        try {

            String eprQuery = createEprQuery(query, refinefields, fieldQueries);

            epr = index.getBrowsingStatistics(eprQuery, "all", mdFormat, indexLayout);

            if (epr == null) {

                throw new SearchServiceException("Something really strange happened there! Index returned null result set id.");

            }

            rs = rsFactory.createResultSet(epr);

            ((SolrResultSet)rs).cursorGet(transformer,os);

        } catch (IndexServiceException ise) {

            logger.error("Error getting cursor results.", ise);

            throw new SearchServiceException("Error getting refine results.", ise);

        } catch (IOException ioe) {

            logger.error("Error getting cursor results.", ioe);

            throw new SearchServiceException("Error getting refine results.", ioe);

        } catch (SolrServerException sse) {

            logger.error("Error getting cursor results.", sse);

            throw new SearchServiceException("Error getting refine results.", sse);

        }

        long estimatedTime = System.nanoTime() - startTime;

        logger.info("Search time " + estimatedTime/1000000 +  " milliseconds for cursor query " + query +

                " and fields " + fieldQueries + " and refine " + refinefields);

        rs.close();

    }

    /*

    public OutputStream cursorSearch(String text, List<String> refinefields, List<String> fieldQueries) throws SearchServiceException, IOException, SolrServerException {

        //logger.info("newSearch > from: " + from + " to:" + to);

        long startTime = System.nanoTime();

        IndexService index = getIndexLocator().getService();

        EPR epr = null;

        ResultSet<String> rs = null;

        List<String> browseResults = null;

        List<String> searchResults = null;

        String query = rewrite(text);

        enhanceFieldQueries(fieldQueries);

        logger.info("Performing query " + query + "' and fields " + fieldQueries + " and refine " + refinefields);

        try {

            //TODO see parser and maybe delete!

            //query = new CQLParser().parse(query).toCQL();

            String eprQuery = createEprQuery(query, refinefields, fieldQueries);

            epr = index.getBrowsingStatistics(eprQuery, "all", mdFormat, indexLayout);

            if (epr == null) {

                throw new SearchServiceException("Something really strange happened there! Index returned null result set id.");

            }

            //get the locale TODO do we need this?

            //String correctLocale = getCorrectLocale(locale);

            //StringTokenizer tokenizer = new StringTokenizer(correctLocale, "_");

            //Locale requestLocale = new Locale(tokenizer.nextToken(), tokenizer.nextToken());

            rs = rsFactory.createResultSet(epr);

            Map<String, List<String>> list = null;

            return ((SolrResultSet)rs).cursorGet(transformer);

        } catch (IndexServiceException ise) {

            logger.error("Error getting refine results.", ise);

            throw new SearchServiceException("Error getting refine results.", ise);

        }

    }*/

import javax.servlet.http.HttpServletResponse;

import javax.ws.rs.core.StreamingOutput;

import java.io.*;

import java.util.HashMap;

import java.util.Map;

import java.util.concurrent.TimeUnit;

    @GET

    @Path("/reports")

    @Produces(MediaType.TEXT_PLAIN)

    public Response fetchReport(@QueryParam("query") String query,

                                @QueryParam("refine") @DefaultValue("false") boolean refine,

                                @QueryParam("fields") final List<String> fields,

                                @QueryParam("fq") final List<String> fieldQueries,

                                @QueryParam("type") final String type,

                                @QueryParam("format") final String format,

                                @Context final HttpServletResponse response) throws Exception {

        RequestResponseHandler.Entity entity =  extractEntity(type);

        Transformer transformer = extractTransformer(format, extractEntity(type));

        boolean special = isSpecialFormat(format);

        String responseType = extractResponseFormat(format);

        StreamingOutput stream = new StreamingOutput() {

            @Override

            public void write(OutputStream os) throws IOException, WebApplicationException

            {

                try {

                    os.write(CSVResponseFormat.appendTitle(entity, special).getBytes());

                    searchService.cursorSearch(query, fields, fieldQueries, format, transformer, os);

                } catch (SearchServiceException sse) {

                    os.write(("Fail to return report." + sse.getMessage()).getBytes());

                    logger.error("Fail to return report.", sse);

                }

                os.close();

            }

        };

        return Response.ok().entity(stream).type( MediaType.TEXT_PLAIN).build();

        //q=*:*&start=0&rows=10&cursorMark=*&sort=dateofcollection asc

/*        try {

            queryOpts.add("q", new CqlTranslatorImpl().getTranslatedQuery("(oaftype exact result) and (resulttypeid exact publication)").asLucene());

        } catch (CQLParseException e) {

            logger.error(e);

        }

        queryOpts.add("start", "0");

        queryOpts.add("rows", "500");

        queryOpts.add("fl", "__result");

        queryOpts.add("shards.tolerant","true");

        queryOpts.add("cursorMark", "*");

        queryOpts.add("sort", "__indexrecordidentifier asc");

        StreamingOutput stream = new StreamingOutput() {

            @Override

            public void write(OutputStream os) throws IOException, WebApplicationException

            {

                int curs = 0;

                QueryResponse resp = null;

                String cursorMark = "*";

                String nextCursorMark = "";

                while (!cursorMark.equals(nextCursorMark)) {

                    logger.debug("QUERY OPTS: " + queryOpts.get("cursorMark"));

                    try {

                        resp = solrClient.query(SolrParams.toSolrParams(queryOpts));

                    } catch (SolrServerException e) {

                        logger.error("ERROROROROROROR!! ", e);

                        e.printStackTrace();

                    }

                    logger.debug("TOTAL number " + resp.getResults().getNumFound());

                    logger.debug(resp.getNextCursorMark());

                    System.out.println("BEGIN");

                    System.out.println("cursor " + cursorMark);

                    System.out.println("next cursor " + nextCursorMark);

                    cursorMark = nextCursorMark;

                    nextCursorMark = resp.getNextCursorMark();

                    for (int i = 0; i < resp.getResults().size(); i++) {

                        String result = ((ArrayList<String>) resp.getResults().get(i).get("__result")).get(0);

                        os.write(result.getBytes());

                        os.flush();

                    }

                    System.out.println("END");

                    System.out.println("cursor " + cursorMark);

                    System.out.println("next cursor " + nextCursorMark);

                    queryOpts.remove("cursorMark");

                    queryOpts.add("cursorMark", nextCursorMark);

                    System.out.println("CURS " + curs);

                    curs ++;

                }

            }

        };

        //out.close();

        time = System.currentTimeMillis() - time;

        System.out.println("Answer time " + time);

        //StreamingOutput output = response.getWriter();

        return Response.ok().entity( stream ).type( MediaType.TEXT_PLAIN).build();*/

        //return Response.status(Response.Status.OK).build();

    }

    @GET

    @Path("/testreports")

    @Produces(MediaType.TEXT_PLAIN)

    public Response loadHierarchy(@PathParam( "pkPerson" ) String pkPerson) {

        final Map<Integer, String> people  = new HashMap<>();

        people.put(1, "Michael\n");

        people.put(2, "Mark\n");

        StreamingOutput stream = new StreamingOutput() {

            @Override

            public void write(OutputStream os) throws IOException, WebApplicationException

            {

                while (true) {

                    Writer writer = new BufferedWriter(new OutputStreamWriter(os));

                    writer.write(people.get(1));

                    writer.flush();

                    try {

                        TimeUnit.SECONDS.sleep(1);

                    } catch (InterruptedException e) {

                        e.printStackTrace();

                        logger.error("ERROR.", e);

                    }

                }

                /*while(true) {

                    os.write(people.get(1).getBytes());

                    os.flush();

                }*/

            }

        };

        return Response.ok().entity( stream ).type( MediaType.TEXT_PLAIN).build()    ;

    }

    private static final String SOFTWARE_BASIC_QUERY = "(oaftype exact result) and (resulttypeid exact software)";

        long time = System.currentTimeMillis();

        time = System.currentTimeMillis() - time;

        System.out.println("Answer old time " + time);

    @RequestMapping(value = "/api/software", method = RequestMethod.GET)

    public void searchSoftware(HttpServletRequest request, HttpServletResponse response) {

        PrintWriter writer = null;

        try {

            writer = response.getWriter();

            checkParameters(ListUtils.union(PUB_N_DATA_COMMON_PARAMETERS, DATA_PARAMETERS),request.getParameterMap());

            int page = readParameter(request, "page", 1);

            int size = readParameter(request, "size", 10);

            checkRequestSize(page, size);

            String format = (request.getParameter("format") != null) ? request.getParameter("format") : "xml";

            createResponseMeta(response, format);

            checkFormatParameter(PUB_N_DATASET_FORMATS, format);

            String model = request.getParameter("model");

            checkModelParameter(PUB_N_DATASET_MODELS, model);

            String sTransformer = defineTransformer(model,format);

            Collection<String> referrers = readParameter(request,"referrer");

            String newFormat = defineFormatter(model, format, false, referrers);

            String locale = request.getParameter("locale");

            StringBuilder queryBuilder = new StringBuilder();

            queryBuilder.append(SOFTWARE_BASIC_QUERY);

            ParameterQueryEnhancer.enhanceQueryWithFundingLevelParams(queryBuilder, request, vocabularyManager, isModelSygma(model));

            ParameterQueryEnhancer.enhanceQueryWithOpenAIREIds(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithMetadataKeywords(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithFundingParams(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithRelProjectParams(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithAccessRights(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithDate(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithDoi(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithResultsSortParameters(queryBuilder, request);

            FormattedSearchResult formattedSearchResult =  searchService.search(queryBuilder.toString(),sTransformer, (newFormat!=null)?newFormat:format, locale, page, size);

            writer.append(formattedSearchResult.getFormattedResult());

        } catch (Exception e) {

            logger.error("Fail to execute search.", e);

            createXmlErrorPage(writer, e);

        } finally {

            if (writer != null) {

                IOUtils.closeQuietly(writer);

            }

        }

    }

    /*public void test(HttpServletRequest request, HttpServletResponse response) throws IOException, SolrServerException {

        long time = System.currentTimeMillis();

        CloudSolrServer solrClient = new CloudSolrServer("openaire-solr-beta.vls.icm.edu.pl:9983");

        solrClient.setDefaultCollection("DMF-index-openaire");

        response.setContentType("text/html");

        ServletOutputStream out=response.getOutputStream();

        NamedList<String> queryOpts = new NamedList<String>();

        //q=*:*&start=0&rows=10&cursorMark=*&sort=dateofcollection asc

        try {

            queryOpts.add("q", new CqlTranslatorImpl().getTranslatedQuery("(oaftype exact result) and (resulttypeid exact publication) and (relfundershortname exact \"nhmrc_______::NHMRC||National Health and Medical Research Council (NHMRC)||NHMRC\")").asLucene());

        } catch (CQLParseException e) {

            logger.error(e);

        }

        queryOpts.add("start", "0");

        queryOpts.add("rows", "500");

        queryOpts.add("fl", "__result");

        queryOpts.add("shards.tolerant","true");

        queryOpts.add("cursorMark", "*");

        queryOpts.add("sort", "__indexrecordidentifier asc");

        //queryOpts.add("q", new CqlTranslatorImpl().getTranslatedQuery("oaftype exact project").asLucene());

        NamedList<String> extraOpts = new NamedList<String>();

        QueryResponse resp = null;

        String cursorMark = "*";

        String nextCursorMark = "";

        //QueryResponse resp = solrClient.query(SolrParams.toSolrParams(queryOpts));

        int curs = 0;

        while (!cursorMark.equals(nextCursorMark)) {

            logger.debug("QUERY OPTS: " + queryOpts.get("cursorMark"));

            resp = solrClient.query(SolrParams.toSolrParams(queryOpts));

            logger.debug("TOTAL number " + resp.getResults().getNumFound());

            logger.debug(resp.getNextCursorMark());

            System.out.println("BEGIN");

            System.out.println("cursor " + cursorMark);

            System.out.println("next cursor " + nextCursorMark);

            cursorMark = nextCursorMark;

            nextCursorMark = resp.getNextCursorMark();

            for (int i = 0; i < resp.getResults().size(); i++) {

                String result = ((ArrayList<String>) resp.getResults().get(i).get("__result")).get(0);

                out.write(result.getBytes());

                out.flush();

            }

            System.out.println("END");

            System.out.println("cursor " + cursorMark);

            System.out.println("next cursor " + nextCursorMark);

            queryOpts.remove("cursorMark");

            queryOpts.add("cursorMark", nextCursorMark);

            System.out.println("CURS " + curs);

            curs ++;

        }

        out.close();

        time = System.currentTimeMillis() - time;

        System.out.println("Answer time " + time);

    }*/