D-Net

package eu.dnetlib.data.search.web.api;

import com.google.common.collect.Iterables;

import eu.dnetlib.api.data.SearchService;

import eu.dnetlib.api.data.SearchServiceException;

import eu.dnetlib.data.search.utils.cql.ParameterQueryEnhancer;

import eu.dnetlib.data.search.utils.vocabulary.VocabularyManager;

import eu.dnetlib.domain.data.FormattedSearchResult;

import io.micrometer.core.annotation.Timed;

import io.micrometer.prometheus.PrometheusMeterRegistry;

import org.apache.commons.lang.IncompleteArgumentException;

import org.apache.commons.lang.StringEscapeUtils;

import org.apache.log4j.Logger;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.context.annotation.EnableAspectJAutoProxy;

import org.springframework.http.HttpStatus;

import org.springframework.http.MediaType;

import org.springframework.http.ResponseEntity;

import org.springframework.stereotype.Controller;

import org.springframework.ui.ModelMap;

import org.springframework.web.bind.annotation.*;

import javax.annotation.PostConstruct;

import javax.annotation.Resource;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

import java.io.PrintWriter;

import java.text.SimpleDateFormat;

import java.util.*;

@Controller

@EnableAspectJAutoProxy

@Timed

public class SearchRequestController {

    @Autowired

    private SearchService searchService = null;

    @Autowired

    private VocabularyManager vocabularyManager = null;

    @Resource

    private String maxResults = null;

    @Resource

    private String maxSize = null;

    @Autowired

    private PrometheusMeterRegistry registry;

    private static Logger logger = Logger.getLogger(SearchRequestController.class);

    private static final String XML_CONTENT_TYPE = "application/xml;charset=UTF-8";

    private static final String JSON_CONTENT_TYPE = "application/json;charset=UTF-8;";

    private static final String CSV_CONTENT_TYPE = "text/csv;charset=UTF-8;";

    private static final String TSV_CONTENT_TYPE = "text/tab-separated-values;charset=UTF-8;";

    private static final String HTML_CONTENT_TYPE = "text/html;charset=UTF-8;";

    private static final String PUBLICATION_BASIC_QUERY = "(oaftype exact result) and (resulttypeid exact publication)";

    private static final String DATASET_BASIC_QUERY = "(oaftype exact result) and (resulttypeid exact dataset)";

    private static final String SOFTWARE_BASIC_QUERY = "(oaftype exact result) and (resulttypeid exact software)";

    private static final String OTHER_BASIC_QUERY = "(oaftype exact result) and (resulttypeid exact other)";

    private static final String PROJECT_BASIC_QUERY = "(oaftype exact project)";

    private static final HashSet<String> GLOBAL_PARAMETERS = new HashSet<>(Arrays.asList("page", "size", "format", "sortBy"));

    private static final HashSet<String> PUB_N_DATA_COMMON_PARAMETERS = new HashSet<>(Arrays.asList("author", "doi", "originalId",

            "community", "FP7ProjectID", "FP7scientificArea", "fromDateAccepted", "funder", "fundingStream", "hasECFunding", "hasProject",

            "hasWTFunding", "keywords", "model", "OA", "openaireProjectID", "openaireProviderID", "projectID", "title", "toDateAccepted",

            "country", "orcid"));

    private static final HashSet<String> PUB_PARAMETERS = new HashSet<>(Arrays.asList("openairePublicationID"));

    private static final HashSet<String> DATA_PARAMETERS = new HashSet<>(Arrays.asList("openaireDatasetID"));

    private static final HashSet<String> SOFTWARE_PARAMETERS = new HashSet<>(Arrays.asList("openaireSoftwareID"));

    private static final HashSet<String> OTHER_PARAMETERS = new HashSet<>(Arrays.asList("openaireOtherID"));

    private static final HashSet<String> PUB_N_DATASET_MODELS = new HashSet<>(Arrays.asList("dc", "openaire", "sygma"));

    private static final HashSet<String> PUB_N_DATASET_FORMATS = new HashSet<>(Arrays.asList("json", "rss", "xml", "csv", "tsv", "html"));

    private static final HashSet<String> PROJECT_PARAMETERS = new HashSet<>(Arrays.asList("acronym", "callID", "endYear", "FP7scientificArea",

            "funder", "fundingStream", "grantID", "hasECFunding", "hasWTFunding", "keywords", "name",

            "participantAcronyms", "participantCountries", "startYear", "sc39", "openaireParticipantID", "openaireProjectID"));

    private static final HashSet<String> PROJECT_FORMATS = new HashSet<>(Arrays.asList("xml", "json", "csv", "tsv", "html"));

    @PostConstruct

    public void init() {

        PUB_PARAMETERS.addAll(PUB_N_DATA_COMMON_PARAMETERS);

        DATA_PARAMETERS.addAll(PUB_N_DATA_COMMON_PARAMETERS);

        SOFTWARE_PARAMETERS.addAll(PUB_N_DATA_COMMON_PARAMETERS);

        OTHER_PARAMETERS.addAll(PUB_N_DATA_COMMON_PARAMETERS);

    }

    //TODO this is for joomla - to be removed soon

    @RequestMapping(value = "/search", method = RequestMethod.GET)

    @Timed(value = "search.joomla.requests", longTask = false)

    public void search(HttpServletRequest request, HttpServletResponse response) {

        PrintWriter writer = null;

        FormattedSearchResult formattedSearchResult = null;

        String action = readActionParameter(request);

        String query = readQueryParameter(request);

        String format = (request.getParameter("format") != null) ? request.getParameter("format") : "xml";

        createResponseMeta(response, format);

        String locale = request.getParameter("locale");

        int page = 0;

        int size = 0;

        //TODO check paging

        if (!action.equals("refine")) {

            page = readParameter(request, "page", 1);

            size = readParameter(request, "size", 10);

        }

        if (size > Integer.parseInt(maxSize)) {

            throw new IllegalArgumentException("Size argument have exceeded the maximum allowed number.");

        }

        String sTransformer = request.getParameter("sTransformer");

        String rTransformer = request.getParameter("rTransformer");

        Collection<String> fields = readParameter(request, "fields");

        checkTransformerParameters(action, sTransformer, rTransformer, fields);

        try {

            createResponseMeta(response, format);

            writer = response.getWriter();

            formattedSearchResult = searchService.searchNrefine(query,

                    sTransformer, rTransformer, format, locale, page, size,

                    fields);

            writer.append(formattedSearchResult.getFormattedResult());

        } catch (Exception e) {

            logger.error("Fail to execute search.", e);

            createXmlErrorPage(writer, e);

        } finally {

            if (writer != null) {

                writer.close();

            }

        }

    }

    private void checkTransformerParameters(String action, String sTransformer, String rTransformer, Collection<String> fields) {

        if (action.equals("search")) {

            if (sTransformer == null) {

                throw new IncompleteArgumentException("Undefined search transformer. Search request");

            }

        } else if (action.equals("refine")) {

            if (rTransformer == null) {

                throw new IncompleteArgumentException(

                        "Undefined refine transformer. Refine request");

            }

            if (fields == null) {

                throw new IncompleteArgumentException(

                        "Undefined refine fields. Refine request");

            }

        } else if (action.equals("searchNrefine")) {

            if (sTransformer == null) {

                throw new IncompleteArgumentException(

                        "Undefined search transformer. Search and Refine request");

            }

            if (rTransformer == null) {

                throw new IncompleteArgumentException(

                        "Undefined refine transformer. Search and Refine request");

            }

            if (fields == null) {

                throw new IncompleteArgumentException(

                        "Undefined refine fields. Search and Refine request");

            }

        } else {

            throw new UnsupportedOperationException("The action " + action

                    + " is not supported. Please try one of {search, refine, searchNrefine}.");

        }

    }

    private String readQueryParameter(HttpServletRequest request) {

        String query = request.getParameter("query");

        if (query == null) {

            throw new IncompleteArgumentException("Undefined query. Search request");

        }

        return query;

    }

    private String readActionParameter(HttpServletRequest request) {

        String action = request.getParameter("action");

        if (action == null) {

            throw new UnsupportedOperationException("Undefined action.");

        }

        return action;

    }

    @RequestMapping(value="/search/openSearchDescriptor", method = RequestMethod.GET)

    public String printOpenSearchDescriptor(ModelMap model) {

        return "openSearchDescriptor";

    }

    @RequestMapping(value = "/api/publications", method = RequestMethod.GET, produces = {MediaType.APPLICATION_XML_VALUE,MediaType.APPLICATION_JSON_VALUE})

    @Timed(value = "http.server.request.duration", extraTags = {"referer", "api", "uri", "/api/publications"}, longTask = false)

    public void searchPublications(HttpServletRequest request, HttpServletResponse response) throws Exception {

        long time = System.currentTimeMillis();

        int page = readParameter(request, "page", 1);

        int size = readParameter(request, "size", 10);

        String format = (request.getParameter("format") != null) ? request.getParameter("format") : "xml";

        String model = request.getParameter("model");

        checkParameters(PUB_PARAMETERS,request.getParameterMap());

        checkRequestSize(page, size);

        checkFormatParameter(PUB_N_DATASET_FORMATS, format);

        createResponseMeta(response, format);

        checkModelParameter(PUB_N_DATASET_MODELS, model);

        String locale = request.getParameter("locale");

        String sTransformer = defineTransformer(model,format);

        Collection<String> referrers = readParameter(request,"referrer");

        String newFormat = defineFormatter(model, format, true, referrers);

        StringBuilder queryBuilder = new StringBuilder();

        queryBuilder.append(PUBLICATION_BASIC_QUERY);

        enhanceResearchOutcomesQuery(request, model, queryBuilder);

        FormattedSearchResult formattedSearchResult = null;

        try {

            formattedSearchResult = searchService.search(queryBuilder.toString(),sTransformer, (newFormat!=null)?newFormat:format, locale, page, size);

        } catch (Exception e) {

            logger.error("Fail to execute search.", e);

            throw new Exception("Fail to execute search", e);

        }

        PrintWriter writer = response.getWriter();

        writer.append(formattedSearchResult.getFormattedResult());

        writer.close();

        time = System.currentTimeMillis() - time;

        logger.debug("Answer old time " + time);

    }

    private void enhanceResearchOutcomesQuery(HttpServletRequest request, String model, StringBuilder queryBuilder) {

        ParameterQueryEnhancer.enhanceQueryWithFundingLevelParams(queryBuilder, request, vocabularyManager, isModelSygma(model));

        ParameterQueryEnhancer.enhanceQueryWithOpenAIREIds(queryBuilder, request);

        ParameterQueryEnhancer.enhanceQueryWithMetadataKeywords(queryBuilder, request);

        ParameterQueryEnhancer.enhanceQueryWithFundingParams(queryBuilder, request);

        ParameterQueryEnhancer.enhanceQueryWithCommunityParams(queryBuilder, request);

        ParameterQueryEnhancer.enhanceQueryWithRelProjectParams(queryBuilder, request);

        ParameterQueryEnhancer.enhanceQueryWithAccessRights(queryBuilder, request);

        ParameterQueryEnhancer.enhanceQueryWithDate(queryBuilder, request);

        ParameterQueryEnhancer.enhanceQueryWithDoi(queryBuilder, request);

        ParameterQueryEnhancer.enhanceQueryWithOrcid(queryBuilder, request);

        ParameterQueryEnhancer.enhanceQueryWithOriginalId(queryBuilder, request);

        ParameterQueryEnhancer.enhanceQueryWithResultsSortParameters(queryBuilder, request);

    }

    @RequestMapping(value = "/api/datasets", method = RequestMethod.GET, produces = {MediaType.APPLICATION_XML_VALUE,MediaType.APPLICATION_JSON_VALUE})

    @Timed(value = "http.server.request.duration", extraTags = {"referer", "api", "uri", "/api/datasets"}, longTask = false)

    public void searchData(HttpServletRequest request, HttpServletResponse response) throws Exception {

        checkParameters(DATA_PARAMETERS,request.getParameterMap());

        int page = readParameter(request, "page", 1);

        int size = readParameter(request, "size", 10);

        checkRequestSize(page, size);

        String format = (request.getParameter("format") != null) ? request.getParameter("format") : "xml";

        createResponseMeta(response, format);

        checkFormatParameter(PUB_N_DATASET_FORMATS, format);

        String model = request.getParameter("model");

        checkModelParameter(PUB_N_DATASET_MODELS, model);

        String sTransformer = defineTransformer(model,format);

        Collection<String> referrers = readParameter(request,"referrer");

        String newFormat = defineFormatter(model, format, false, referrers);

        String locale = request.getParameter("locale");

        StringBuilder queryBuilder = new StringBuilder();

        queryBuilder.append(DATASET_BASIC_QUERY);

        enhanceResearchOutcomesQuery(request, model, queryBuilder);

        FormattedSearchResult formattedSearchResult =  null;

        try {

            formattedSearchResult = searchService.search(queryBuilder.toString(),sTransformer, (newFormat!=null)?newFormat:format, locale, page, size);

        } catch (Exception e) {

            logger.error("Fail to execute search.", e);

            throw new Exception("Fail to execute search.", e);

        }

        if (formattedSearchResult == null) {

            throw new Exception("Fail to execute search.");

        }

        PrintWriter writer = response.getWriter();

        writer.append(formattedSearchResult.getFormattedResult());

        writer.close();

    }

    @RequestMapping(value = "/api/software", method = RequestMethod.GET, produces = {MediaType.APPLICATION_XML_VALUE,MediaType.APPLICATION_JSON_VALUE})

    @Timed(value = "http.server.request.duration", extraTags = {"referer", "api", "uri", "/api/software"}, longTask = false)

    public void searchSoftware(HttpServletRequest request, HttpServletResponse response) throws Exception {

        checkParameters(SOFTWARE_PARAMETERS,request.getParameterMap());

        int page = readParameter(request, "page", 1);

        int size = readParameter(request, "size", 10);

        checkRequestSize(page, size);

        String format = (request.getParameter("format") != null) ? request.getParameter("format") : "xml";

        createResponseMeta(response, format);

        checkFormatParameter(PUB_N_DATASET_FORMATS, format);

        String model = request.getParameter("model");

        checkModelParameter(PUB_N_DATASET_MODELS, model);

        String sTransformer = defineTransformer(model,format);

        Collection<String> referrers = readParameter(request,"referrer");

        String newFormat = defineFormatter(model, format, false, referrers);

        String locale = request.getParameter("locale");

        StringBuilder queryBuilder = new StringBuilder();

        queryBuilder.append(SOFTWARE_BASIC_QUERY);

        enhanceResearchOutcomesQuery(request, model, queryBuilder);

        FormattedSearchResult formattedSearchResult = null;

        try {

            formattedSearchResult =  searchService.search(queryBuilder.toString(),sTransformer, (newFormat!=null)?newFormat:format, locale, page, size);

        } catch (Exception e) {

            logger.error("Fail to execute search.", e);

            throw new Exception("Fail to execute search", e);

        }

        PrintWriter writer = response.getWriter();

        writer.append(formattedSearchResult.getFormattedResult());

    }

    @RequestMapping(value = "/api/other", method = RequestMethod.GET, produces = {MediaType.APPLICATION_XML_VALUE,MediaType.APPLICATION_JSON_VALUE})

    @Timed(value = "http.server.request.duration", extraTags = {"referer", "api", "uri", "/api/other"}, longTask = false)

    public void searchOther(HttpServletRequest request, HttpServletResponse response) throws Exception {

        checkParameters(OTHER_PARAMETERS,request.getParameterMap());

        int page = readParameter(request, "page", 1);

        int size = readParameter(request, "size", 10);

        checkRequestSize(page, size);

        String format = (request.getParameter("format") != null) ? request.getParameter("format") : "xml";

        createResponseMeta(response, format);

        checkFormatParameter(PUB_N_DATASET_FORMATS, format);

        String model = request.getParameter("model");

        checkModelParameter(PUB_N_DATASET_MODELS, model);

        String sTransformer = defineTransformer(model,format);

        Collection<String> referrers = readParameter(request,"referrer");

        String newFormat = defineFormatter(model, format, false, referrers);

        String locale = request.getParameter("locale");

        StringBuilder queryBuilder = new StringBuilder();

        queryBuilder.append(OTHER_BASIC_QUERY);

        enhanceResearchOutcomesQuery(request, model, queryBuilder);

        FormattedSearchResult formattedSearchResult = null;

        //long start = System.currentTimeMillis();

        try {

            //Timer.Sample sample = Timer.start(metrics.getRegistry());

            formattedSearchResult =  searchService.search(queryBuilder.toString(),sTransformer, (newFormat!=null)?newFormat:format, locale, page, size);

        } catch (Exception e) {

            logger.error("Fail to execute search.", e);

            throw new Exception("Fail to execute search", e);

        }

        PrintWriter writer = response.getWriter();

        writer.append(formattedSearchResult.getFormattedResult());

    }

    @RequestMapping(value = "/api/projects", method = RequestMethod.GET, produces = {MediaType.APPLICATION_XML_VALUE,MediaType.APPLICATION_JSON_VALUE})

    @Timed(value = "http.server.request.duration", extraTags = {"referer", "api", "uri", "/api/projects"}, longTask = false)

    public void searchProjects(HttpServletRequest request, HttpServletResponse response) throws Exception {

        checkParameters(PROJECT_PARAMETERS, request.getParameterMap());

        int page = readParameter(request, "page", 1);

        int size = readParameter(request, "size", 10);

        checkRequestSize(page, size);

        StringBuilder queryBuilder = new StringBuilder();

        queryBuilder.append(PROJECT_BASIC_QUERY);

        String locale = request.getParameter("locale");

        String format =  (request.getParameter("format") != null) ? request.getParameter("format") : "xml";

        checkFormatParameter(PROJECT_FORMATS, format);

        createResponseMeta(response, format);

        format = finalFormat(format);

        String sTransformer = request.getParameter("sTransformer");

        ParameterQueryEnhancer.enhanceProjectQueryWithOpenAIREIds(queryBuilder, request);

        ParameterQueryEnhancer.enhanceQueryWithProjectMetadataKeywords(queryBuilder, request);

        ParameterQueryEnhancer.enhanceQueryWithProjectFundingParams(queryBuilder, request);

        ParameterQueryEnhancer.enhanceProjectQueryWithFundingLevelParams(queryBuilder, request, vocabularyManager);

        ParameterQueryEnhancer.enhanceQueryWithParticipantsInfoParams(queryBuilder, request);

        ParameterQueryEnhancer.enhanceQueryWithYearParams(queryBuilder, request);

        ParameterQueryEnhancer.enhanceQueryWithSC39Params(queryBuilder, request);

        ParameterQueryEnhancer.enhanceQueryWithProjectSortParameters(queryBuilder, request);

        FormattedSearchResult formattedSearchResult = null;

        try {

            formattedSearchResult = searchService.search(queryBuilder.toString(),sTransformer, format, locale, page, size);

        } catch (SearchServiceException e) {

            throw new Exception("Fail to execute search", e);

        }

        PrintWriter writer = response.getWriter();

        writer.append(formattedSearchResult.getFormattedResult());

        writer.close();

    }

    @ExceptionHandler(IllegalArgumentException.class)

    public @ResponseBody ResponseEntity<Error> invalidInput(HttpServletRequest request, HttpServletResponse httpServletResponse, Exception ex) {

        Error response = new Error();

        response.setStatus("error");

        response.setCode("400");

        response.setMessage("400 - Illegal argument exception.");

        response.setException(ex.getMessage());

        String format = (request.getParameter("format") == null)? "xml": request.getParameter("format").toLowerCase();

        registry.counter("http.status.400", "400", "uri").increment();

        return new ResponseEntity<Error>(response, HttpStatus.BAD_REQUEST);

    }

    private String finalFormat(String format) {

        if (format.equals("tsv")){

            return "project_tsv";

        } else if (format.equals("csv")) {

            return "project_csv";

        } else if (format.equals("html")) {

            return "project_html";

        }

        return format;

    }

    private String finalFormat(String format, String referrer) {

        if (referrer == null || referrer.isEmpty() || !referrer.equals("joomla")) {

            return finalFormat(format);

        }

        if (format.equals("tsv")){

            return "publication_tsv_notitle";

        } else if (format.equals("csv")) {

            return "publication_csv_notitle";

        }

        return format;

    }

    /** TODO: check if needed

     * Based on the given model the transformer to be used is defined

     * @param model

     * @return

    private String defineTransformer(String model) {

    if (model!=null && !model.trim().isEmpty()) {

    if (model.equals("openaire")) {

    return null;

    } else if (model.equals("sygma")) {

    return "results_openaire";

    }

    else if (model.equals("dc")) {

    return "dc";

    }

    throw new IllegalArgumentException("model '" + model + "' is not supported.");

    } else {

    return null;

    }

    }*/

    //TODO: check if needed

    private String defineTransformer(String model, String format) {

        if (model != null && !model.trim().isEmpty() && format != null && (format.equals("json")||format.equals("xml"))) {

            if (model.equals("openaire")) {

                return null;

            } else if (model.equals("sygma")) {

                return "results_openaire";

            }

            else if (model.equals("dc")) {

                return "dc";

            }

            throw new IllegalArgumentException("model '" + model + "' is not supported.");

        } else if (format != null && !format.trim().isEmpty()) {

            if (format.equals("rss") || format.equals("csv") || format.equals("tsv") || format.equals("html")) {

                return "results_openaire";

            }

        }

        return null;

    }

    private String defineFormatter(String model, String format, boolean isPublications, Collection<String> referrers) {

        if( model != null && !model.trim().isEmpty() ) {

            if (model.equals("sygma")) {

                if (isPublications) {

                    return "sygma_publication";

                }

                return "sygma_dataset";

            }

        } else {

            if (referrers != null && !referrers.isEmpty() && Iterables.get(referrers, 0) != null && !Iterables.get(referrers, 0).isEmpty()) {

                return finalFormat(format, Iterables.get(referrers, 0));

            } else {

                format = finalFormat(format);

            }

        }

        return null;

    }

    @Deprecated

    private void createXmlErrorPage(PrintWriter writer, Exception e) {

        writer.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");

        writer.append("<error>");

        writer.append("<exception>")

                .append(StringEscapeUtils.escapeXml(e.getClass().getName()))

                .append("</exception>");

        if (e.getMessage() != null) {

            writer.append("<message>")

                    .append(StringEscapeUtils.escapeXml(e.getMessage()))

                    .append("</message>");

        }

        if (e.getCause() != null) {

            writer.append("<cause>")

                    .append(StringEscapeUtils.escapeXml(e.getCause()

                            .toString())).append("</cause>");

        }

        StackTraceElement[] trace = e.getStackTrace();

        writer.append("<trace>");

        for (int i = 0; i < trace.length; i++) {

            writer.append(StringEscapeUtils.escapeXml(trace[i].toString()))

                    .append("\n");

        }

        writer.append("</trace>");

        writer.append("</error>");

    }

    // Reading parameters from url and adding default values

    public int readParameter(HttpServletRequest request, String parameterName, int defaultValue) {

        String param = request.getParameter(parameterName);

        return (param != null && !param.isEmpty()) ? Integer.parseInt(param) : defaultValue;

    }

    private Collection<String> readParameter(HttpServletRequest request, String parameterName) {

        Collection<String> fields = null;

        String[] paramfields = request.getParameterValues(parameterName);

        if (paramfields != null) {

            fields = new ArrayList<String>();

            for (int i = 0; i < paramfields.length; i++) {

                fields.add(paramfields[i]);

            }

        }

        return fields;

    }

    /**

     * Checks if the requested result size exceeds the maximum allowed numbers of returned results

     * @param page the page parameter

     * @param size the size parameter

     */

    private void checkRequestSize(int page, int size) {

        int total = page*size;

        if (total > Integer.parseInt(maxResults)) {

            throw new IllegalArgumentException("Size and page arguments have exceeded the maximum number of returned results.");

        }

    }

    private String defineContentType(String format) {

        if (format != null) {

            if (format.equalsIgnoreCase("xml")) {

                return XML_CONTENT_TYPE;

            } else if (format.equalsIgnoreCase("json")) {

                return JSON_CONTENT_TYPE;

            } else if (format.equalsIgnoreCase("csv")) {

                return CSV_CONTENT_TYPE;

            } else if (format.equalsIgnoreCase("tsv")) {

                return TSV_CONTENT_TYPE;

            } else if (format.equalsIgnoreCase("html")){

                return HTML_CONTENT_TYPE;

            }

        }

        return XML_CONTENT_TYPE;

    }

    /**

     * Checks if the parameters given are allowed. The given parameters are checked against the allowed parameter

     * and the GLOBAL_PARAMETERS.

     * @param allowedParameters the allowed parameters

     * @param currentParameters the given parameters

     */

    private void checkParameters(HashSet<String> allowedParameters, Map<String, String[]> currentParameters){

        if(currentParameters != null) {

            for (String parameter : currentParameters.keySet()) {

                if (!allowedParameters.contains(parameter) && !GLOBAL_PARAMETERS.contains(parameter) && !parameter.equals("referrer")) {

                    throw new IllegalArgumentException("Parameter " + parameter + " is not supported. The supported parameters are: " +

                            allowedParameters.toString().replace("[","").replace("]", ", ") + GLOBAL_PARAMETERS.toString().substring(1,GLOBAL_PARAMETERS.toString().length()-1));

                }

            }

        }

    }

    private void checkFormatParameter(HashSet<String> allowedFormats, String requestedFormat) {

        if (requestedFormat!= null && !allowedFormats.contains(requestedFormat)) {

            throw new IllegalArgumentException("The requested format \'"+ requestedFormat +"\' is not supported. The supported formats are: " + allowedFormats);

        }

    }

    private static void checkModelParameter(HashSet<String> allowedModels, String requestedModel) {

        if (requestedModel!= null && !allowedModels.contains(requestedModel)) {

            throw new IllegalArgumentException("The requested model \'"+ allowedModels +"\' is not supported. The supported formats are: " + allowedModels);

        }

    }

    private static boolean isModelSygma(String model) {

        if (model == null || model.isEmpty()) { return false; }

        if (!model.isEmpty() && !model.equals("sygma")) {

            return  false;

        }

        return true;

    }

    private void createResponseMeta( HttpServletResponse response, String format) {

        response.setContentType(defineContentType(format));

        response.setCharacterEncoding("UTF-8");

        if (!format.equals("xml") || !format.equals("json")) {

            SimpleDateFormat sdf = new SimpleDateFormat("ddMMyyyy");

            String date = sdf.format(new Date());

            if (format.equals("csv")) {

                response.setHeader("Content-Disposition", "filename="+date+".csv");

            } else if (format.equals("csv")) {

                response.setHeader("Content-Disposition", "filename="+date+".tsv");

            } else if (format.equals("html")) {

                response.setHeader("Content-Disposition", "filename="+date+".html");

            }

        }

    }

    /* TODO: enable if we decide to use ACCEPT Header as a priority to format parameter

    private String getFormat(HttpServletRequest request) {

        if (request.getParameter("format") == null && request.getHeader("Accept")!=null) {

            if (request.getHeader("Accept").equals(MediaType.APPLICATION_XML_VALUE) ||

                    request.getHeader("Accept").contains(MediaType.APPLICATION_XML_VALUE)){

                return "xml";

            }

            if (request.getHeader("Accept").equals(MediaType.APPLICATION_JSON_VALUE)){

                return "json";

            }

        }

        if (request.getParameter("format") == null && (request.getHeader("Accept")== null ||

                request.getHeader("Accept").equals(MediaType.ALL_VALUE))) {

            return "xml";

        }

        return request.getParameter("format");

    } */

    /*public static void main() throws IOException, SolrServerException {

        long time = System.currentTimeMillis();

        CloudSolrClient solrClient = new CloudSolrClient.Builder().

                withZkHost(Arrays.asList(new String[]{"quorum0.t.hadoop.research-infrastructures.eu:2182",

                        "quorum1.t.hadoop.research-infrastructures.eu:2182",

                        "quorum2.t.hadoop.research-infrastructures.eu:2182",

                        "quorum3.t.hadoop.research-infrastructures.eu:2182",

                        "quorum4.t.hadoop.research-infrastructures.eu:2182/solr-dev-openaire"})).build();

        solrClient.setDefaultCollection("DMF-index-openaire");

       // response.setContentType("text/html");

       // ServletOutputStream out=response.getOutputStream();

        NamedList<String> queryOpts = new NamedList<String>();

        //q=*:*&start=0&rows=10&cursorMark=*&sort=dateofcollection asc

        try {

            queryOpts.add("q", new CqlTranslatorImpl().getTranslatedQuery("(oaftype exact result) and (resulttypeid exact publication) and (relfundershortname exact \"nhmrc_______::NHMRC||National Health and Medical Research Council (NHMRC)||NHMRC\")").asLucene());

        } catch (CQLParseException e) {

            logger.error(e);

        }

        queryOpts.add("start", "0");

        queryOpts.add("rows", "500");

        queryOpts.add("fl", "__result");

        queryOpts.add("shards.tolerant","true");

        queryOpts.add("cursorMark", "*");

        queryOpts.add("sort", "__indexrecordidentifier asc");

        //queryOpts.add("q", new CqlTranslatorImpl().getTranslatedQuery("oaftype exact project").asLucene());

        NamedList<String> extraOpts = new NamedList<String>();

        QueryResponse resp = null;

        String cursorMark = "*";

        String nextCursorMark = "";

        //QueryResponse resp = solrClient.query(SolrParams.toSolrParams(queryOpts));

        int curs = 0;

        while (!cursorMark.equals(nextCursorMark)) {

            logger.debug("QUERY OPTS: " + queryOpts.get("cursorMark"));

            resp = solrClient.query(SolrParams.toSolrParams(queryOpts));

            logger.debug("TOTAL number " + resp.getResults().getNumFound());

            logger.debug(resp.getNextCursorMark());

            System.out.println("BEGIN");

            System.out.println("cursor " + cursorMark);

            System.out.println("next cursor " + nextCursorMark);