D-Net

package eu.dnetlib.data.search.web.api;

import com.google.common.collect.Iterables;

import eu.dnetlib.data.search.app.SearchServiceImpl;

import eu.dnetlib.data.search.utils.cql.ParameterQueryEnhancer;

import eu.dnetlib.data.search.utils.vocabulary.VocabularyManager;

import eu.dnetlib.domain.data.FormattedSearchResult;

import org.apache.commons.collections.ListUtils;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang.IncompleteArgumentException;

import org.apache.commons.lang.StringEscapeUtils;

import org.apache.log4j.Logger;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.stereotype.Controller;

import org.springframework.ui.ModelMap;

import org.springframework.web.bind.annotation.RequestMapping;

import org.springframework.web.bind.annotation.RequestMethod;

import javax.annotation.Resource;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

import java.io.PrintWriter;

import java.text.SimpleDateFormat;

import java.util.*;

@Controller

public class SearchRequestController {

	@Autowired

	private SearchServiceImpl searchService = null;

	@Autowired

	private VocabularyManager vocabularyManager = null;

	@Resource

	private String maxResults = null;

	@Resource

	private String maxSize = null;

	private static Logger logger = Logger.getLogger(SearchRequestController.class);

	private static final String XML_CONTENT_TYPE = "application/xml;charset=UTF-8";

	private static final String JSON_CONTENT_TYPE = "application/json;charset=UTF-8;";

    private static final String CSV_CONTENT_TYPE = "text/csv;charset=UTF-8;";

    private static final String TSV_CONTENT_TYPE = "text/tab-separated-values;charset=UTF-8;";

    private static final String HTML_CONTENT_TYPE = "text/html;charset=UTF-8;";

    private static final String PUBLICATION_BASIC_QUERY = "(oaftype exact result) and (resulttypeid exact publication)";

    private static final String DATASET_BASIC_QUERY = "(oaftype exact result) and (resulttypeid exact dataset)";

    private static final String SOFTWARE_BASIC_QUERY = "(oaftype exact result) and (resulttypeid exact software)";

    private static final String PROJECT_BASIC_QUERY = "(oaftype exact project)";

    private static final List<String> GLOBAL_PARAMETERS = Arrays.asList("page", "size", "format", "sortBy");

    private static final List<String> PUB_N_DATA_COMMON_PARAMETERS = Arrays.asList("author", "doi", "FP7ProjectID", "FP7scientificArea",

            "fromDateAccepted", "funder", "fundingStream", "hasECFunding", "hasProject", "hasWTFunding", "keywords",

            "model", "OA", "openaireAuthorID", "openaireProjectID", "openaireProviderID", "projectID", "title", "toDateAccepted");

    private static final List<String> PUB_PARAMETERS = Arrays.asList("openairePublicationID");

    private static final List<String> DATA_PARAMETERS = Arrays.asList("openaireDatasetID");

    private static final List<String> SOFTWARE_PARAMETERS = Arrays.asList("openaireSoftwareID");

    private static final List<String> PUB_N_DATASET_MODELS = Arrays.asList("dc", "openaire", "sygma");

    private static final List<String> PUB_N_DATASET_FORMATS = Arrays.asList("json", "rss", "xml", "csv", "tsv", "html");

    private static final List<String> PROJECT_PARAMETERS = Arrays.asList("acronym", "callID", "endYear", "FP7scientificArea",

            "funder", "fundingStream", "grantID", "hasECFunding", "hasWTFunding", "keywords", "name",

            "participantAcronyms", "participantCountries", "startYear", "sc39", "openaireParticipantID", "openaireProjectID");

    private static final List<String> PROJECT_FORMATS = Arrays.asList("xml", "json", "csv", "tsv", "html");

	@RequestMapping(value = "/search", method = RequestMethod.GET)

	public void search(HttpServletRequest request, HttpServletResponse response) {

		PrintWriter writer = null;

		FormattedSearchResult formattedSearchResult = null;

        String action = readActionParameter(request);

        String query = readQueryParameter(request);

		String format = (request.getParameter("format") != null) ? request.getParameter("format") : "xml";

        createResponseMeta(response, format);

		String locale = request.getParameter("locale");

        int page= 0;

        int size =-1;

        //TODO check paging

        if (!action.equals("refine")) {

            page = readParameter(request, "page", 1);

            size = readParameter(request, "size", 10);

        }

		if (size > Integer.parseInt(maxSize)) {

			throw new IllegalArgumentException("Size argument have exceeded the maximum allowed number.");

		}

		String sTransformer = request.getParameter("sTransformer");

		String rTransformer = request.getParameter("rTransformer");

		Collection<String> fields = readParameter(request, "fields");

        checkTransformerParameters(action, sTransformer, rTransformer, fields);

        try {

            createResponseMeta(response, format);

            writer = response.getWriter();

            formattedSearchResult = searchService.searchNrefine(query,

                    sTransformer, rTransformer, format, locale, page, size,

                    fields);

			writer.append(formattedSearchResult.getFormattedResult());

		} catch (Exception e) {

			logger.error("Fail to execute search.", e);

			createXmlErrorPage(writer, e);

		} finally {

            if (writer != null) {

                writer.close();

            }

		}

	}

    private void checkTransformerParameters(String action, String sTransformer, String rTransformer, Collection<String> fields) {

        if (action.equals("search")) {

            if (sTransformer == null) {

                throw new IncompleteArgumentException("Undefined search transformer. Search request");

            }

        } else if (action.equals("refine")) {

            if (rTransformer == null) {

                throw new IncompleteArgumentException(

                        "Undefined refine transformer. Refine request");

            }

            if (fields == null) {

                throw new IncompleteArgumentException(

                        "Undefined refine fields. Refine request");

            }

        } else if (action.equals("searchNrefine")) {

            if (sTransformer == null) {

                throw new IncompleteArgumentException(

                        "Undefined search transformer. Search and Refine request");

            }

            if (rTransformer == null) {

                throw new IncompleteArgumentException(

                        "Undefined refine transformer. Search and Refine request");

            }

            if (fields == null) {

                throw new IncompleteArgumentException(

                        "Undefined refine fields. Search and Refine request");

            }

        } else {

            throw new UnsupportedOperationException("The action " + action

                    + " is not supported. Please try one of {search, refine, searchNrefine}.");

        }

    }

    private String readQueryParameter(HttpServletRequest request) {

        String query = request.getParameter("query");

        if (query == null) {

            throw new IncompleteArgumentException("Undefined query. Search request");

        }

        return query;

    }

    private String readActionParameter(HttpServletRequest request) {

        String action = request.getParameter("action");

        if (action == null) {

            throw new UnsupportedOperationException("Undefined action.");

        }

        return action;

    }

    @RequestMapping(value="/search/openSearchDescriptor", method = RequestMethod.GET)

    public String printOpenSearchDescriptor(ModelMap model) {

        return "openSearchDescriptor";

    }

	@RequestMapping(value = "/api/publications", method = RequestMethod.GET)

	public void searchPublications(HttpServletRequest request, HttpServletResponse response) {

        long time = System.currentTimeMillis();

		PrintWriter writer = null;

        try {

            writer = response.getWriter();

            checkParameters(ListUtils.union(PUB_N_DATA_COMMON_PARAMETERS, PUB_PARAMETERS),request.getParameterMap());

            int page = readParameter(request, "page", 1);

            int size = readParameter(request, "size", 10);

            checkRequestSize(page, size);

            String format = (request.getParameter("format") != null) ? request.getParameter("format") : "xml";

            checkFormatParameter(PUB_N_DATASET_FORMATS, format);

            createResponseMeta(response, format);

            String locale = request.getParameter("locale");

            String model = request.getParameter("model");

            checkModelParameter(PUB_N_DATASET_MODELS, model);

            String sTransformer = defineTransformer(model,format);

            Collection<String> referrers = readParameter(request,"referrer");

            String newFormat = defineFormatter(model, format, true, referrers);

            StringBuilder queryBuilder = new StringBuilder();

            queryBuilder.append(PUBLICATION_BASIC_QUERY);

			ParameterQueryEnhancer.enhanceQueryWithFundingLevelParams(queryBuilder, request, vocabularyManager, isModelSygma(model));

            ParameterQueryEnhancer.enhanceQueryWithOpenAIREIds(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithMetadataKeywords(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithFundingParams(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithRelProjectParams(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithAccessRights(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithDate(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithDoi(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithResultsSortParameters(queryBuilder, request);

			FormattedSearchResult formattedSearchResult = searchService.search(queryBuilder.toString(),sTransformer, (newFormat!=null)?newFormat:format, locale, page, size);

			writer.append(formattedSearchResult.getFormattedResult());

		} catch (Exception e) {

			logger.error("Fail to execute search.", e);

			createXmlErrorPage(writer, e);

		} finally {

            if (writer != null) {

                IOUtils.closeQuietly(writer);

            }

		}

        time = System.currentTimeMillis() - time;

        System.out.println("Answer old time " + time);

	}

	@RequestMapping(value = "/api/datasets", method = RequestMethod.GET)

	public void searchData(HttpServletRequest request, HttpServletResponse response) {

		PrintWriter writer = null;

        try {

            writer = response.getWriter();

            checkParameters(ListUtils.union(PUB_N_DATA_COMMON_PARAMETERS, DATA_PARAMETERS),request.getParameterMap());

            int page = readParameter(request, "page", 1);

            int size = readParameter(request, "size", 10);

            checkRequestSize(page, size);

            String format = (request.getParameter("format") != null) ? request.getParameter("format") : "xml";

            createResponseMeta(response, format);

            checkFormatParameter(PUB_N_DATASET_FORMATS, format);

            String model = request.getParameter("model");

            checkModelParameter(PUB_N_DATASET_MODELS, model);

            String sTransformer = defineTransformer(model,format);

            Collection<String> referrers = readParameter(request,"referrer");

            String newFormat = defineFormatter(model, format, false, referrers);

            String locale = request.getParameter("locale");

            StringBuilder queryBuilder = new StringBuilder();

            queryBuilder.append(DATASET_BASIC_QUERY);

            ParameterQueryEnhancer.enhanceQueryWithFundingLevelParams(queryBuilder, request, vocabularyManager, isModelSygma(model));

            ParameterQueryEnhancer.enhanceQueryWithOpenAIREIds(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithMetadataKeywords(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithFundingParams(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithRelProjectParams(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithAccessRights(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithDate(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithDoi(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithResultsSortParameters(queryBuilder, request);

			FormattedSearchResult formattedSearchResult =  searchService.search(queryBuilder.toString(),sTransformer, (newFormat!=null)?newFormat:format, locale, page, size);

			writer.append(formattedSearchResult.getFormattedResult());

		} catch (Exception e) {

			logger.error("Fail to execute search.", e);

			createXmlErrorPage(writer, e);

		} finally {

            if (writer != null) {

                IOUtils.closeQuietly(writer);

            }

		}

	}

    @RequestMapping(value = "/api/software", method = RequestMethod.GET)

    public void searchSoftware(HttpServletRequest request, HttpServletResponse response) {

        PrintWriter writer = null;

        try {

            writer = response.getWriter();

            checkParameters(ListUtils.union(PUB_N_DATA_COMMON_PARAMETERS, SOFTWARE_PARAMETERS),request.getParameterMap());

            int page = readParameter(request, "page", 1);

            int size = readParameter(request, "size", 10);

            checkRequestSize(page, size);

            String format = (request.getParameter("format") != null) ? request.getParameter("format") : "xml";

            createResponseMeta(response, format);

            checkFormatParameter(PUB_N_DATASET_FORMATS, format);

            String model = request.getParameter("model");

            checkModelParameter(PUB_N_DATASET_MODELS, model);

            String sTransformer = defineTransformer(model,format);

            Collection<String> referrers = readParameter(request,"referrer");

            String newFormat = defineFormatter(model, format, false, referrers);

            String locale = request.getParameter("locale");

            StringBuilder queryBuilder = new StringBuilder();

            queryBuilder.append(SOFTWARE_BASIC_QUERY);

            ParameterQueryEnhancer.enhanceQueryWithFundingLevelParams(queryBuilder, request, vocabularyManager, isModelSygma(model));

            ParameterQueryEnhancer.enhanceQueryWithOpenAIREIds(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithMetadataKeywords(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithFundingParams(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithRelProjectParams(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithAccessRights(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithDate(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithDoi(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithResultsSortParameters(queryBuilder, request);

            FormattedSearchResult formattedSearchResult =  searchService.search(queryBuilder.toString(),sTransformer, (newFormat!=null)?newFormat:format, locale, page, size);

            writer.append(formattedSearchResult.getFormattedResult());

        } catch (Exception e) {

            logger.error("Fail to execute search.", e);

            createXmlErrorPage(writer, e);

        } finally {

            if (writer != null) {

                IOUtils.closeQuietly(writer);

            }

        }

    }

	@RequestMapping(value = "/api/projects", method = RequestMethod.GET)

	public void searchProjects(HttpServletRequest request, HttpServletResponse response)  {

		PrintWriter writer = null;

        try {

            writer = response.getWriter();

            checkParameters(PROJECT_PARAMETERS, request.getParameterMap());

            int page = readParameter(request, "page", 1);

            int size = readParameter(request, "size", 10);

            checkRequestSize(page, size);

            StringBuilder queryBuilder = new StringBuilder();

            queryBuilder.append(PROJECT_BASIC_QUERY);

            String locale = request.getParameter("locale");

            String format = (request.getParameter("format") != null) ? request.getParameter("format") : "xml";

            checkFormatParameter(PROJECT_FORMATS, format);

            createResponseMeta(response, format);

            format = finalFormat(format);

            String sTransformer = request.getParameter("sTransformer");

            FormattedSearchResult formattedSearchResult = null;

            ParameterQueryEnhancer.enhanceProjectQueryWithOpenAIREIds(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithProjectMetadataKeywords(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithProjectFundingParams(queryBuilder, request);

            ParameterQueryEnhancer.enhanceProjectQueryWithFundingLevelParams(queryBuilder, request, vocabularyManager);

            ParameterQueryEnhancer.enhanceQueryWithParticipantsInfoParams(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithYearParams(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithSC39Params(queryBuilder, request);

            ParameterQueryEnhancer.enhanceQueryWithProjectSortParameters(queryBuilder, request);

			formattedSearchResult = searchService.search(queryBuilder.toString(),sTransformer, format, locale, page, size);

			writer.append(formattedSearchResult.getFormattedResult());

		} catch (Exception e) {

			logger.error("Fail to execute search.", e);

            createXmlErrorPage(writer, e);

		} finally {

            if (writer != null) {

                IOUtils.closeQuietly(writer);

            }

		}

	}

    private String finalFormat(String format) {

        if (format.equals("tsv")){

            return "project_tsv";

        } else if (format.equals("csv")) {

            return "project_csv";

        } else if (format.equals("html")) {

            return "project_html";

        }

        return format;

    }

    private String finalFormat(String format, String referrer) {

        if (referrer == null || referrer.isEmpty() || !referrer.equals("joomla")) {

            return finalFormat(format);

        }

        if (format.equals("tsv")){

            return "publication_tsv_notitle";

        } else if (format.equals("csv")) {

            return "publication_csv_notitle";

        }

        return format;

    }

    /** TODO: check if needed

     * Based on the given model the transformer to be used is defined

     * @param model

     * @return

    private String defineTransformer(String model) {

        if (model!=null && !model.trim().isEmpty()) {

            if (model.equals("openaire")) {

                return null;

            } else if (model.equals("sygma")) {

                return "results_openaire";

            }

            else if (model.equals("dc")) {

                return "dc";

            }

            throw new IllegalArgumentException("model '" + model + "' is not supported.");

        } else {

            return null;

        }

    }*/

    //TODO: check if needed

 private String defineTransformer(String model, String format) {

   if (model!=null && !model.trim().isEmpty()) {

            if (model.equals("openaire")) {

                return null;

            } else if (model.equals("sygma")) {

                return "results_openaire";

            }

            else if (model.equals("dc")) {

                return "dc";

            }

            throw new IllegalArgumentException("model '" + model + "' is not supported.");

        } else if (format!=null && !format.trim().isEmpty()) {

            if (format.equals("rss") || format.equals("csv") || format.equals("tsv") || format.equals("html")) {

			    return "results_openaire";

		    }

        } 

		return null;

    }

    private String defineFormatter(String model, String format, boolean isPublications, Collection<String> referrers) {

        if( model != null && !model.trim().isEmpty() ) {

            if (model.equals("sygma")) {

                if (isPublications) {

                    return "sygma_publication";

                }

                return "sygma_dataset";

            }

        } else {

            if (referrers != null && !referrers.isEmpty() && Iterables.get(referrers, 0) != null && !Iterables.get(referrers, 0).isEmpty()) {

                return finalFormat(format, Iterables.get(referrers, 0));

            } else {

                format = finalFormat(format);

            }

        }

        return null;

    }

    private void createXmlErrorPage(PrintWriter writer, Exception e) {

		writer.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");

		writer.append("<error>");

		writer.append("<exception>")

				.append(StringEscapeUtils.escapeXml(e.getClass().getName()))

				.append("</exception>");

		if (e.getMessage() != null) {

			writer.append("<message>")

					.append(StringEscapeUtils.escapeXml(e.getMessage()))

					.append("</message>");

		}

		if (e.getCause() != null) {

			writer.append("<cause>")

					.append(StringEscapeUtils.escapeXml(e.getCause()

							.toString())).append("</cause>");

		}

		StackTraceElement[] trace = e.getStackTrace();

		writer.append("<trace>");

		for (int i = 0; i < trace.length; i++) {

			writer.append(StringEscapeUtils.escapeXml(trace[i].toString()))

					.append("\n");

		}

		writer.append("</trace>");

		writer.append("</error>");

	}

    // Reading parameters from url and adding default values

    public int readParameter(HttpServletRequest request, String parameterName, int defaultValue) {

        String param = request.getParameter(parameterName);

        return (param != null && !param.isEmpty()) ? Integer.parseInt(param) : defaultValue;

    }

    private Collection<String> readParameter(HttpServletRequest request, String parameterName) {

        Collection<String> fields = null;

        String[] paramfields = request.getParameterValues(parameterName);

        if (paramfields != null) {

            fields = new ArrayList<String>();

            for (int i = 0; i < paramfields.length; i++) {

                fields.add(paramfields[i]);

            }

        }

        return fields;

    }

    /**

	 * Checks if the requested result size exceeds the maximum allowed numbers of returned results

	 * @param page the page parameter

	 * @param size the size parameter

	 */

	private void checkRequestSize(int page, int size) {

		int total = page*size;

		if (total > Integer.parseInt(maxResults)) {

			throw new IllegalArgumentException("Size and page arguments have exceeded the maximum number of returned results.");

		}

	}

    private String defineContentType(String format) {

        if (format != null) {

            if (format.equalsIgnoreCase("xml")) {

                return XML_CONTENT_TYPE;

            } else if (format.equalsIgnoreCase("json")) {

                return JSON_CONTENT_TYPE;

			} else if (format.equalsIgnoreCase("csv")) {

                return CSV_CONTENT_TYPE;

            } else if (format.equalsIgnoreCase("tsv")) {

                return TSV_CONTENT_TYPE;

            } else if (format.equalsIgnoreCase("html")){

                return HTML_CONTENT_TYPE;

            }

        }

        return XML_CONTENT_TYPE;

    }

    /**

     * Checks if the parameters given are allowed. The given parameters are checked against the allowed parameter

     * and the GLOBAL_PARAMETERS.

     * @param allowedParameters the allowed parameters

     * @param currentParameters the given parameters

     */

    private void checkParameters(List<String> allowedParameters, Map<String, String[]> currentParameters){

        if(currentParameters != null) {

            for (String parameter : currentParameters.keySet()) {

                if (!allowedParameters.contains(parameter) && !GLOBAL_PARAMETERS.contains(parameter) && !parameter.equals("referrer")) {

                    throw new IllegalArgumentException("Parameter " + parameter + " is not supported. The supported parameters are: " +

                            allowedParameters.toString().replace("[","").replace("]", ", ") + GLOBAL_PARAMETERS.toString().substring(1,GLOBAL_PARAMETERS.toString().length()-1));

                }

            }

        }

    }

    private void checkFormatParameter(List<String> allowedFormats, String requestedFormat) {

        if (requestedFormat!= null && !allowedFormats.contains(requestedFormat)) {

            throw new IllegalArgumentException("The requested format \'"+ requestedFormat +"\' is not supported. The supported formats are: " + allowedFormats);

        }

    }

    private static void checkModelParameter(List<String> allowedModels, String requestedModel) {

        if (requestedModel!= null && !allowedModels.contains(requestedModel)) {

            throw new IllegalArgumentException("The requested model \'"+ allowedModels +"\' is not supported. The supported formats are: " + allowedModels);

        }

    }

    private static boolean isModelSygma(String model) {

        if (model == null || model.isEmpty()) { return false; }

        if (!model.isEmpty() && !model.equals("sygma")) {

            return  false;

        }

        return true;

    }

    private void createResponseMeta( HttpServletResponse response, String format) {

        response.setContentType(defineContentType(format));

        response.setCharacterEncoding("UTF-8");

        if (!format.equals("xml") || !format.equals("json")) {

            SimpleDateFormat sdf = new SimpleDateFormat("ddMMyyyy");

            String date = sdf.format(new Date());

            if (format.equals("csv")) {

                response.setHeader("Content-Disposition", "filename="+date+".csv");

            } else if (format.equals("csv")) {

                response.setHeader("Content-Disposition", "filename="+date+".tsv");

            } else if (format.equals("html")) {

                response.setHeader("Content-Disposition", "filename="+date+".html");

            }

        }

    }

    /*public void test(HttpServletRequest request, HttpServletResponse response) throws IOException, SolrServerException {

        long time = System.currentTimeMillis();

        CloudSolrServer solrClient = new CloudSolrServer("openaire-solr-beta.vls.icm.edu.pl:9983");

        solrClient.setDefaultCollection("DMF-index-openaire");

        response.setContentType("text/html");

        ServletOutputStream out=response.getOutputStream();

        NamedList<String> queryOpts = new NamedList<String>();

        //q=*:*&start=0&rows=10&cursorMark=*&sort=dateofcollection asc

        try {

            queryOpts.add("q", new CqlTranslatorImpl().getTranslatedQuery("(oaftype exact result) and (resulttypeid exact publication) and (relfundershortname exact \"nhmrc_______::NHMRC||National Health and Medical Research Council (NHMRC)||NHMRC\")").asLucene());

        } catch (CQLParseException e) {

            logger.error(e);

        }

        queryOpts.add("start", "0");

        queryOpts.add("rows", "500");

        queryOpts.add("fl", "__result");

        queryOpts.add("shards.tolerant","true");

        queryOpts.add("cursorMark", "*");

        queryOpts.add("sort", "__indexrecordidentifier asc");

        //queryOpts.add("q", new CqlTranslatorImpl().getTranslatedQuery("oaftype exact project").asLucene());

        NamedList<String> extraOpts = new NamedList<String>();

        QueryResponse resp = null;

        String cursorMark = "*";

        String nextCursorMark = "";

        //QueryResponse resp = solrClient.query(SolrParams.toSolrParams(queryOpts));

        int curs = 0;

        while (!cursorMark.equals(nextCursorMark)) {

            logger.debug("QUERY OPTS: " + queryOpts.get("cursorMark"));

            resp = solrClient.query(SolrParams.toSolrParams(queryOpts));

            logger.debug("TOTAL number " + resp.getResults().getNumFound());

            logger.debug(resp.getNextCursorMark());

            System.out.println("BEGIN");

            System.out.println("cursor " + cursorMark);

            System.out.println("next cursor " + nextCursorMark);

            cursorMark = nextCursorMark;

            nextCursorMark = resp.getNextCursorMark();

            for (int i = 0; i < resp.getResults().size(); i++) {

                String result = ((ArrayList<String>) resp.getResults().get(i).get("__result")).get(0);

                out.write(result.getBytes());

                out.flush();

            }

            System.out.println("END");

            System.out.println("cursor " + cursorMark);

            System.out.println("next cursor " + nextCursorMark);

            queryOpts.remove("cursorMark");

            queryOpts.add("cursorMark", nextCursorMark);

            System.out.println("CURS " + curs);

            curs ++;

        }

        out.close();

        time = System.currentTimeMillis() - time;

        System.out.println("Answer time " + time);

    }*/

}