D-Net

package eu.dnetlib.data.search.app.plan.web.api;

import eu.dnetlib.data.search.app.SearchServiceImpl;

import eu.dnetlib.data.search.solr.SolrResultSet;

import eu.dnetlib.data.search.utils.SolrResultSetOptionsUtil;

import eu.dnetlib.data.search.web.api.SearchApiService;

import eu.dnetlib.data.search.web.utils.RequestResponseHandler;

import eu.dnetlib.functionality.index.cql.CqlTranslator;

import eu.dnetlib.functionality.index.cql.CqlTranslatorImpl;

import org.apache.log4j.BasicConfigurator;

import org.apache.solr.client.solrj.impl.CloudSolrServer;

import org.apache.solr.common.util.NamedList;

import org.junit.Assert;

import org.junit.Before;

import org.junit.Test;

import org.mockito.Mockito;

import org.z3950.zing.cql.CQLParseException;

import javax.servlet.http.HttpServletRequest;

import java.io.IOException;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.List;

import static org.mockito.Matchers.anyString;

import static org.mockito.Mockito.doNothing;

/**

 * Created by kiatrop on 9/11/2016.

 */

public class SolrUtilsTest {

    private org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(SearchApiService.class);

    HttpServletRequest requestMock;

    RequestResponseHandler.Entity entity;

    List<String> refineFields = new ArrayList<String>();

    List<String> fieldQueries = new ArrayList<String>();

    CloudSolrServer mockSolrClient;

    SolrResultSet mockRS;

    @Before

    public  void before() {

        BasicConfigurator.configure();

/*      requestMock = Mockito.mock(HttpServletRequest.class);

        Map<String, String[]> mockParameterMap = new HashMap<String, String[]>();

        mockParameterMap.put("fq", new String[]{"(relfunderid exact ec___::EC) and (relfunderid exact wt___::WT)",

                "relfundinglevel0_id exact ec___::EC::SP1", "(relfundinglevel1_id exact ec___::EC::SP1::VALUE1) or (relfundinglevel1_id exact ec___::EC::SP1::VALUE2)"});

        when(requestMock.getParameterMap()).thenReturn(mockParameterMap);

        when(requestMock.getParameterValues("fq")).thenReturn(new String[]{"(relfunderid exact ec___::EC) and (relfunderid exact wt___::WT)",

                "relfundinglevel0_id exact ec___::EC::SP1" , "(relfundinglevel1_id exact ec___::EC::SP1::VALUE1) or (relfundinglevel1_id exact ec___::EC::SP1::VALUE2)"});

*/

        mockSolrClient = Mockito.mock(CloudSolrServer.class);

        doNothing().when(mockSolrClient).setDefaultCollection(anyString());

        mockRS = Mockito.mock(SolrResultSet.class);

        refineFields = Arrays.asList(new String[]{"relfunderid", "relfundinglevel0_id", "relfundinglevel1_id", "relfundinglevel2_id"});

        fieldQueries = Arrays.asList(new String[]{"(relfunderid exact ec___::EC) and (relfunderid exact wt___::WT)",

                "relfundinglevel0_id exact ec___::EC::SP1" , "(relfundinglevel1_id exact ec___::EC::SP1::VALUE1) or (relfundinglevel1_id exact ec___::EC::SP1::VALUE2)"});

    }

    @Test

    public void createEprQuery() {

        String query = SearchServiceImpl.createEprQuery("(oaftype=result)", refineFields, fieldQueries);

        Assert.assertEquals("query=(oaftype=result)" +

                "&groupby=relfunderid,relfundinglevel0_id,relfundinglevel1_id,relfundinglevel2_id" +

                "&fq=(relfunderid exact ec___::EC) and (relfunderid exact wt___::WT),relfundinglevel0_id exact ec___::EC::SP1," +

                        "(relfundinglevel1_id exact ec___::EC::SP1::VALUE1) or (relfundinglevel1_id exact ec___::EC::SP1::VALUE2)", query);

        //empty refine, empty fq

        query = SearchServiceImpl.createEprQuery("(oaftype=result)", new ArrayList<String>(), new ArrayList<String>());

        Assert.assertEquals("query=(oaftype=result)" +

                "&groupby=" +

                "&fq=", query);

        //empty fq

        query = SearchServiceImpl.createEprQuery("(oaftype=result)", refineFields, new ArrayList<String>());

        Assert.assertEquals("query=(oaftype=result)" +

                "&groupby=relfunderid,relfundinglevel0_id,relfundinglevel1_id,relfundinglevel2_id" +

                "&fq=", query);

        //empty refine

        query = SearchServiceImpl.createEprQuery("(oaftype=result)", new ArrayList<String>(), fieldQueries);

        Assert.assertEquals("query=(oaftype=result)" +

                "&groupby=" +

                "&fq=(relfunderid exact ec___::EC) and (relfunderid exact wt___::WT),relfundinglevel0_id exact ec___::EC::SP1," +

                "(relfundinglevel1_id exact ec___::EC::SP1::VALUE1) or (relfundinglevel1_id exact ec___::EC::SP1::VALUE2)", query);

        //empty query

        query = SearchServiceImpl.createEprQuery("", new ArrayList<String>(), new ArrayList<String>());

        Assert.assertEquals("query=" +

                "&groupby=" +

                "&fq=", query);

        //null values

        query = SearchServiceImpl.createEprQuery(null, null, null);

        Assert.assertEquals("query=" +

                "&groupby=" +

                "&fq=", query);

    }

    @Test

    public void extractQueryOptions() throws IOException, CQLParseException {

        CqlTranslator cqlTranslator = new CqlTranslatorImpl();

        NamedList<String> actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", refineFields, fieldQueries));

        Assert.assertEquals(cqlTranslator.toLucene("(oaftype=result)"), actual.get("q"));

        Assert.assertEquals("__result", actual.get("fl"));

        Assert.assertEquals("AND", actual.get("q.op"));

        Assert.assertEquals("true", actual.get("shards.tolerant"));

        Assert.assertEquals("true", actual.get("facet"));

        Assert.assertEquals("1", actual.get("facet.mincount"));

        Assert.assertEquals(refineFields.size() + "", actual.get("facet.threads"));

        Assert.assertEquals(refineFields.size(), actual.getAll("facet.field").size());

        Assert.assertTrue(actual.getAll("facet.field").contains("relfunderid"));

        Assert.assertTrue(actual.getAll("facet.field").contains("relfundinglevel1_id"));

        Assert.assertTrue(actual.getAll("facet.field").contains("relfundinglevel2_id"));

        Assert.assertFalse(actual.getAll("facet.field").contains("relfundinglevel3_id"));

        Assert.assertEquals(fieldQueries.size(), actual.getAll("fq").size());

        Assert.assertTrue(actual.getAll("fq").contains(cqlTranslator.toLucene("(relfunderid exact ec___::EC) and (relfunderid exact wt___::WT)")));

        Assert.assertTrue(actual.getAll("fq").contains(cqlTranslator.toLucene("relfundinglevel0_id exact ec___::EC::SP1")));

        Assert.assertTrue(actual.getAll("fq").contains(cqlTranslator.toLucene("(relfundinglevel1_id exact ec___::EC::SP1::VALUE1) or (relfundinglevel1_id exact ec___::EC::SP1::VALUE2)")));

        //empty refine, empty fq

        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", new ArrayList<String>(), new ArrayList<String>()));

        Assert.assertEquals(cqlTranslator.toLucene("(oaftype=result)"), actual.get("q"));

        Assert.assertEquals("__result", actual.get("fl"));

        Assert.assertNull(actual.get("q.op"));

        Assert.assertEquals("true", actual.get("shards.tolerant"));

        Assert.assertNull(actual.get("facet"));

        Assert.assertNull(actual.get("facet.mincount"));

        Assert.assertNull(actual.get("facet.threads"));

        Assert.assertNull(actual.get("facet.field"));

        Assert.assertTrue(actual.getAll("facet.field").isEmpty());

        Assert.assertNull(actual.get("fq"));

        Assert.assertTrue(actual.getAll("fq").isEmpty());

        //empty fq

        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", refineFields, new ArrayList<String>()));

        Assert.assertEquals(cqlTranslator.toLucene("(oaftype=result)"), actual.get("q"));

        Assert.assertEquals("__result", actual.get("fl"));

        Assert.assertNull(actual.get("q.op"));

        Assert.assertEquals("true", actual.get("shards.tolerant"));

        Assert.assertEquals("true", actual.get("facet"));

        Assert.assertEquals("1", actual.get("facet.mincount"));

        Assert.assertEquals(refineFields.size() + "", actual.get("facet.threads"));

        Assert.assertEquals(refineFields.size(), actual.getAll("facet.field").size());

        Assert.assertTrue(actual.getAll("facet.field").contains("relfunderid"));

        Assert.assertTrue(actual.getAll("facet.field").contains("relfundinglevel1_id"));

        Assert.assertTrue(actual.getAll("facet.field").contains("relfundinglevel2_id"));

        Assert.assertFalse(actual.getAll("facet.field").contains("relfundinglevel3_id"));

        Assert.assertNull(actual.get("fq"));

        Assert.assertTrue(actual.getAll("fq").isEmpty());

        //null fq

        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", refineFields, null));

        Assert.assertEquals(cqlTranslator.toLucene("(oaftype=result)"), actual.get("q"));

        Assert.assertEquals("__result", actual.get("fl"));

        Assert.assertNull(actual.get("q.op"));

        Assert.assertEquals("true", actual.get("shards.tolerant"));

        Assert.assertEquals("true", actual.get("facet"));

        Assert.assertEquals("1", actual.get("facet.mincount"));

        Assert.assertEquals(refineFields.size() + "", actual.get("facet.threads"));

        Assert.assertEquals(refineFields.size(), actual.getAll("facet.field").size());

        Assert.assertTrue(actual.getAll("facet.field").contains("relfunderid"));

        Assert.assertTrue(actual.getAll("facet.field").contains("relfundinglevel1_id"));

        Assert.assertTrue(actual.getAll("facet.field").contains("relfundinglevel2_id"));

        Assert.assertFalse(actual.getAll("facet.field").contains("relfundinglevel3_id"));

        Assert.assertNull(actual.get("fq"));

        Assert.assertTrue(actual.getAll("fq").isEmpty());

        //empty refine

        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", new ArrayList<String>(), fieldQueries));

        Assert.assertEquals(cqlTranslator.toLucene("(oaftype=result)"), actual.get("q"));

        Assert.assertEquals("__result", actual.get("fl"));

        Assert.assertEquals("AND", actual.get("q.op"));

        Assert.assertEquals("true", actual.get("shards.tolerant"));

        Assert.assertNull(actual.get("facet"));

        Assert.assertNull(actual.get("facet.mincount"));

        Assert.assertNull(actual.get("facet.threads"));

        Assert.assertNull(actual.get("facet.field"));

        Assert.assertTrue(actual.getAll("facet.field").isEmpty());

        Assert.assertEquals(fieldQueries.size(), actual.getAll("fq").size());

        Assert.assertTrue(actual.getAll("fq").contains(cqlTranslator.toLucene("(relfunderid exact ec___::EC) and (relfunderid exact wt___::WT)")));

        Assert.assertTrue(actual.getAll("fq").contains(cqlTranslator.toLucene("relfundinglevel0_id exact ec___::EC::SP1")));

        Assert.assertTrue(actual.getAll("fq").contains(cqlTranslator.toLucene("(relfundinglevel1_id exact ec___::EC::SP1::VALUE1) or (relfundinglevel1_id exact ec___::EC::SP1::VALUE2)")));

        //null refine

        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", null, fieldQueries));

        Assert.assertEquals(cqlTranslator.toLucene("(oaftype=result)"), actual.get("q"));

        Assert.assertEquals("__result", actual.get("fl"));

        Assert.assertEquals("AND", actual.get("q.op"));

        Assert.assertEquals("true", actual.get("shards.tolerant"));

        Assert.assertNull(actual.get("facet"));

        Assert.assertNull(actual.get("facet.mincount"));

        Assert.assertNull(actual.get("facet.threads"));

        Assert.assertNull(actual.get("facet.field"));

        Assert.assertTrue(actual.getAll("facet.field").isEmpty());

        Assert.assertEquals(fieldQueries.size(), actual.getAll("fq").size());

        Assert.assertTrue(actual.getAll("fq").contains(cqlTranslator.toLucene(("(relfunderid exact ec___::EC) and (relfunderid exact wt___::WT)"))));

        Assert.assertTrue(actual.getAll("fq").contains(cqlTranslator.toLucene("relfundinglevel0_id exact ec___::EC::SP1")));

        Assert.assertTrue(actual.getAll("fq").contains(cqlTranslator.toLucene("(relfundinglevel1_id exact ec___::EC::SP1::VALUE1) or (relfundinglevel1_id exact ec___::EC::SP1::VALUE2)")));

        //null refine, null fq

        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", null, null));

        Assert.assertEquals(cqlTranslator.toLucene("(oaftype=result)"), actual.get("q"));

        Assert.assertEquals("__result", actual.get("fl"));

        Assert.assertNull(actual.get("q.op"));

        Assert.assertEquals("true", actual.get("shards.tolerant"));

        Assert.assertNull(actual.get("facet"));

        Assert.assertNull(actual.get("facet.mincount"));

        Assert.assertNull(actual.get("facet.threads"));

        Assert.assertNull(actual.get("facet.field"));

        Assert.assertTrue(actual.getAll("facet.field").isEmpty());

        Assert.assertNull(actual.get("fq"));

        Assert.assertTrue(actual.getAll("fq").isEmpty());

    }

    @Test(expected=CQLParseException.class)

    public void extractQueryOptionsEmptyQuery() throws IOException, CQLParseException {

        NamedList<String> actual  = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery(null, null, null));

    }

}

package eu.dnetlib.data.search.utils;

import org.apache.log4j.Logger;

import org.json.XML;

import java.util.Iterator;

import java.util.List;

/**

 * Created by kiatrop on 16/11/2016.

 */

public class SolrResultsFormatter {

    private static final Logger logger = Logger.getLogger(SolrResultsFormatter.class);

    public static String jsonList2Json(List<String> jsons) {

        StringBuilder entitesBuilder = new StringBuilder();

        for(Iterator<String> jsonIterator = jsons.iterator(); jsonIterator.hasNext();){

            entitesBuilder.append(jsonIterator.next());

            if(jsonIterator.hasNext()){

                entitesBuilder.append(",");

            }

        }

        return entitesBuilder.toString();

    }

    public static String xml2Json(String resultXml) {

        StringBuilder entitesBuilder = new StringBuilder();

        if (resultXml!= null) {

            return XML.toJSONObject(resultXml).toString();

        }

        return "";

    }

}

package eu.dnetlib.data.search.utils;

import eu.dnetlib.functionality.index.cql.CqlTranslator;

import eu.dnetlib.functionality.index.cql.CqlTranslatorImpl;

import eu.dnetlib.functionality.index.cql.TranslatedQuery;

import org.apache.log4j.Logger;

import org.apache.solr.common.util.NamedList;

import org.z3950.zing.cql.CQLParseException;

import java.io.IOException;

/**

 * Created by kiatrop on 21/11/2016.

 */

public class SolrResultSetOptionsUtil {

    private static final Logger logger = Logger.getLogger(SolrResultSetOptionsUtil.class);

    public static NamedList<String> extractQueryOptions(String eprQuery) throws CQLParseException, IOException {

        CqlTranslator translator = new CqlTranslatorImpl();

        NamedList<String> queryOpts = new NamedList<String>();

        String[] queryParts = eprQuery.split("&groupby=");

        TranslatedQuery translatedQuery = translator.getTranslatedQuery(queryParts[0].replace("query=",""));

        queryOpts.add("q", translatedQuery.asLucene());

        queryOpts.add("fl", "__result");

        queryOpts.add("shards.tolerant","true");

        if (translatedQuery.getOptions() != null && translatedQuery.getOptions().getSort()!= null  ) {

            queryOpts.add("sort", translatedQuery.getOptions().getSort().getField() + " " + translatedQuery.getOptions().getSort().getMode());

        }

        if (queryParts.length > 1) {

            String[] facetParts = queryParts[1].split("&fq=");

            String[] refineParts = null;

            String[] facetQueries = null;

            if (facetParts != null && facetParts.length > 0) {

                if (!facetParts[0].isEmpty()) {

                    refineParts = facetParts[0].split(",");

                }

                if (facetParts.length > 1){

                    facetQueries = facetParts[1].split(",");

                }

                if (facetParts[0].isEmpty() && facetParts.length > 1) {

                    facetQueries = facetParts[1].split(",");

                }

            }

            if (refineParts != null && refineParts.length > 0) {

                queryOpts.add("facet", "true");

                queryOpts.add("facet.mincount", "1");

                queryOpts.add("facet.threads", refineParts.length + "");

                for (String field : refineParts) {

                    queryOpts.add("facet.field", field);

                }

            }

            if (facetQueries!=null && facetQueries.length > 0) {

                queryOpts.add("q.op", "AND");

                for (String facetPart : facetQueries) {

                    queryOpts.add("fq", translator.toLucene(facetPart));

                }

            }

        }

        return queryOpts;

    }

}

package eu.dnetlib.data.search.web.servlet;

import eu.dnetlib.api.data.SearchService;

import eu.dnetlib.data.search.app.SearchServiceImpl;

import eu.dnetlib.domain.data.FormattedSearchResult;

import org.apache.commons.lang.IncompleteArgumentException;

import org.apache.commons.lang.StringEscapeUtils;

import org.apache.log4j.Logger;

import org.springframework.context.ApplicationContext;

import org.springframework.web.context.support.WebApplicationContextUtils;

import javax.servlet.ServletConfig;

import javax.servlet.ServletException;

import javax.servlet.http.HttpServlet;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

import java.io.PrintWriter;

import java.util.ArrayList;

import java.util.Collection;

@SuppressWarnings("serial")

public class SearchServlet extends HttpServlet{

	private SearchServiceImpl searchService = null;

	private ApplicationContext context = null;

	private static Logger logger = Logger.getLogger(SearchService.class); 

	public void init(ServletConfig config) throws ServletException {

		super.init(config);

		context = WebApplicationContextUtils.getWebApplicationContext(getServletContext());

		searchService = (SearchServiceImpl) context.getBean("searchService");

	}

	@Override

	public void doGet(HttpServletRequest request, HttpServletResponse response) {		

		PrintWriter writer = null;

		FormattedSearchResult formattedSearchResult = null;

		String action = request.getParameter("action");

		if (action == null) {

			throw new UnsupportedOperationException("Undefined action.");

		}

		String query = request.getParameter("query");

		if (query == null) {

			throw new IncompleteArgumentException("Undefined query. Search request");

		}

		String format = (request.getParameter("format") != null) ?

				request.getParameter("format") : "xml";

		String locale = request.getParameter("locale");

		int page = readParameter(request, "page", 1); 					

		int size = readParameter(request, "size", 10);

		String sTransformer = request.getParameter("sTransformer");

		String rTransformer = request.getParameter("rTransformer");

		Collection<String> fields = readParameter(request, "fields");

		response.setContentType("text/xml;charset=UTF-8");

		try {			

			writer = response.getWriter();	

			if(action.equals("search")) {				

				if (sTransformer == null) {

					throw new IncompleteArgumentException("Undefined search transformer. Search request");

				}				

				formattedSearchResult = searchService.search(query, sTransformer, format, locale, page, size);

			} else if(action.equals("refine")) {

				if(rTransformer == null) {

					throw new IncompleteArgumentException("Undefined refine transformer. Refine request");

				}

				if(fields == null) {

					throw new IncompleteArgumentException("Undefined refine fields. Refine request");

				}

				formattedSearchResult = searchService.refine(query, rTransformer, format, locale, fields);

			} else if(action.equals("searchNrefine")) {								

				if(sTransformer == null) {

					throw new IncompleteArgumentException("Undefined search transformer. Search and Refine request");

				}

				if(rTransformer == null) {

					throw new IncompleteArgumentException("Undefined refine transformer. Search and Refine request");

				}

				if(fields == null) {

					throw new IncompleteArgumentException("Undefined refine fields. Search and Refine request");

				}

				formattedSearchResult = searchService.searchNrefine(query, sTransformer, rTransformer, format, locale, page, size, fields);

			} else {

				throw new UnsupportedOperationException("The action " + action + " is not supported.");

			}

			writer.append(formattedSearchResult.getFormattedResult());	

		} catch (Exception e) {			

			writer.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");

			writer.append("<error>");

			writer.append("<exception>").append(StringEscapeUtils.escapeXml(e.getClass().getName())).append("</exception>");

			if (e.getMessage() != null) {

				writer.append("<message>").append(StringEscapeUtils.escapeXml(e.getMessage())).append("</message>");

			}

			if (e.getCause() != null) {

				writer.append("<cause>").append(StringEscapeUtils.escapeXml(e.getCause().toString())).append("</cause>");

			}

			StackTraceElement[] trace = e.getStackTrace();

			writer.append("<trace>");

			for (int i = 0; i < trace.length; i++) {

				writer.append(StringEscapeUtils.escapeXml(trace[i].toString())).append("\n");

			}

			writer.append("</trace>");

			writer.append("</error>");

		} finally {

			if (writer != null) {

				writer.close();

			}

		}

	}

	private Collection<String> readParameter(HttpServletRequest request, String parameterName) {

		Collection<String> fields = null;

		String[] paramfields = request.getParameterValues(parameterName);

		if (paramfields != null) {

			fields = new ArrayList<String>(); 

			for (int i=0; i<paramfields.length; i++) {

				fields.add(paramfields[i]);

			}

		}

		return fields;

	}

	public int readParameter(HttpServletRequest request, String parameterName, int defaultValue) {

		String param = request.getParameter(parameterName);

		return (param != null)?Integer.parseInt(param):defaultValue;

	}

}

package eu.dnetlib.data.search.web.api;

import eu.dnetlib.data.search.utils.vocabulary.VocabularyManager;

import eu.dnetlib.domain.enabling.Vocabulary;

import org.apache.log4j.Logger;

import javax.servlet.http.HttpServletRequest;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.Date;

import java.util.Locale;

/**

 * Created by kiatrop on 10/7/2014.

 */

public class QueryEnhancer {

    private static final Logger logger = Logger.getLogger(QueryEnhancer.class);

    /**

     * Enhance the given CQL query with FP7 specific index fields

     * @param queryBuilder

     * @param request

     */

    public static void enhanceQueryWithFundingLevelParams(StringBuilder queryBuilder, HttpServletRequest request, VocabularyManager vocabularyManager, boolean isModelSygma) {

        logger.debug("enhance funding " + isModelSygma);

        String funder = request.getParameter("funder");

        String fundingStream = request.getParameter("fundingStream");

        String FP7scientificArea = request.getParameter("FP7scientificArea");

        if(isModelSygma) {

            if (funder != null) {

                if(!funder.equalsIgnoreCase("WT")) {

                    Vocabulary sygmaFundersVocabulary = vocabularyManager.getVocabulary("sygma_funders", Locale.ROOT);

                    addExactQueryTerm("relfundinglevel0_id", devocabularizedTerm(funder.toUpperCase(), sygmaFundersVocabulary), queryBuilder);

                } else {

                    Vocabulary funderVocabulary = vocabularyManager.getVocabulary("funders_simple", Locale.ROOT);

                    addExactQueryTerm("relfunderid", devocabularizedTerm(funder.toUpperCase(), funderVocabulary), queryBuilder);

                }

            }

            if (fundingStream != null && !fundingStream.trim().isEmpty()) {

                Vocabulary relfundinglevel1Vocabulary = vocabularyManager.getVocabulary("programmes_simple", Locale.ROOT);

                queryBuilder.append(" and (relfundinglevel1_id exact \"").append(devocabularizedTerm(fundingStream.toUpperCase(), relfundinglevel1Vocabulary)).append("\")");

            }

            if (FP7scientificArea != null && !FP7scientificArea.trim().isEmpty()) {

                Vocabulary relfundinglevel2Vocabulary = vocabularyManager.getVocabulary("areas", Locale.ROOT);

                queryBuilder.append(" and (relfundinglevel2_id exact \"").append(devocabularizedTerm(FP7scientificArea, relfundinglevel2Vocabulary)).append("\")");

            }

        } else {

            if (funder != null) {

                Vocabulary funderVocabulary = vocabularyManager.getVocabulary("funders_simple", Locale.ROOT);

                addExactQueryTerm("relfunderid", devocabularizedTerm(funder.toUpperCase(), funderVocabulary), queryBuilder);

            }

            if (fundingStream != null && !fundingStream.trim().isEmpty()) {

                addORQueryTerm(new String[]{"relfundinglevel0_name", "relfundinglevel1_name", "relfundinglevel2_name"}, fundingStream.toUpperCase(), queryBuilder);

            }

            if (FP7scientificArea != null && !FP7scientificArea.trim().isEmpty()) {

                Vocabulary relfundinglevel2Vocabulary = vocabularyManager.getVocabulary("areas", Locale.ROOT);

                queryBuilder.append(" and (relfundinglevel2_id exact \"").append(devocabularizedTerm(FP7scientificArea, relfundinglevel2Vocabulary)).append("\")");

            }

        }

    }

    public static void enhanceProjectQueryWithFundingLevelParams(StringBuilder queryBuilder, HttpServletRequest request, VocabularyManager vocabularyManager) {

        String funder = request.getParameter("funder");

        String fundingStream = request.getParameter("fundingStream");

        String FP7scientificArea = request.getParameter("FP7scientificArea");

        if (funder != null) {

            //addExactQueryTerm("fundinglevel0_name", funder.toUpperCase(), queryBuilder);

            Vocabulary funderVocabulary = vocabularyManager.getVocabulary("funders_simple", Locale.ROOT);

            addExactQueryTerm("funderid", devocabularizedTerm(funder.toUpperCase(), funderVocabulary), queryBuilder);

        }

        if (fundingStream != null && !fundingStream.trim().isEmpty()) {

            //addExactQueryTerm("fundinglevel1_name", fundingStream.toUpperCase(), queryBuilder);

            addORQueryTerm(new String[]{"fundinglevel0_name", "fundinglevel1_name", "fundinglevel2_name"}, fundingStream.toUpperCase(), queryBuilder);

        }

        if (FP7scientificArea != null && !FP7scientificArea.trim().isEmpty()) {

            Vocabulary relfundinglevel2Vocabulary = vocabularyManager.getVocabulary("areas", Locale.ROOT);

            queryBuilder.append(" and (fundinglevel2_id exact \"").append(devocabularizedTerm(FP7scientificArea, relfundinglevel2Vocabulary)).append("\")");

        }

    }

    public static void enhanceQueryWithFundingParams(StringBuilder queryBuilder, HttpServletRequest request) {

        String hasECFunding = request.getParameter("hasECFunding");

        String hasWTFunding = request.getParameter("hasWTFunding");

        addBooleanQueryTerm("contextid", hasECFunding, "EC", queryBuilder);

        addBooleanQueryTerm("contextid", hasWTFunding, "WT", queryBuilder);

    }

    public static void enhanceQueryWithProjectFundingParams(StringBuilder queryBuilder, HttpServletRequest request) {

        String hasECFunding = request.getParameter("hasECFunding");

        String hasWTFunding = request.getParameter("hasWTFunding");

        addBooleanQueryTerm("funderid", hasECFunding, "ec__________::EC", queryBuilder);

        addBooleanQueryTerm("funderid", hasWTFunding, "wt__________::WT", queryBuilder);

    }

    public static void enhanceQueryWithRelProjectParams(StringBuilder queryBuilder, HttpServletRequest request) {

        String hasProject = request.getParameter("hasProject");

        String projectID = request.getParameter("projectID");

        String FP7ProjectID = request.getParameter("FP7ProjectID");

        if (hasProject != null && !hasProject.isEmpty()) {

            if (hasProject.equals("true")) {

                addEqualQueryTerm("relprojectid", "*", queryBuilder);

            } else {

                addNotEqualQueryTerm("relprojectid", "*", queryBuilder);

            }

        }

        if (FP7ProjectID != null && !FP7ProjectID.trim().isEmpty()) {

            addExactQueryTerm("relprojectcode", FP7ProjectID, queryBuilder);

            addExactQueryTerm(" relfundinglevel0_id", "ec__________::EC::FP7", queryBuilder);

        }

        if (projectID != null && !projectID.trim().isEmpty()) {

            queryBuilder.append(" and (relprojectcode exact \"").append(projectID).append("\")");

        }

    }

    public static void enhanceQueryWithAccessRights(StringBuilder queryBuilder, HttpServletRequest request) {

        String oa = request.getParameter("OA");

        addBooleanQueryTerm("resultbestlicense", oa, "Open Access", queryBuilder);

    }

    public static void enhanceQueryWithDate(StringBuilder queryBuilder, HttpServletRequest request) throws IllegalArgumentException{

        String fromDateAccepted = request.getParameter("fromDateAccepted");

        String toDateAccepted = request.getParameter("toDateAccepted");

        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");

        simpleDateFormat.setLenient(false);

        if (toDateAccepted != null && !toDateAccepted.isEmpty() && (fromDateAccepted == null || fromDateAccepted.isEmpty()) ) {

            if(!checkDate(toDateAccepted, simpleDateFormat)){

                //logger.debug("Format is " + !checkDate(toDateAccepted, simpleDateFormat));

                throw new IllegalArgumentException("toDateAccepted date must be formatted as YYYY-MM-DD");

            }

            fromDateAccepted = "*";

        }

        if (fromDateAccepted != null && !fromDateAccepted.isEmpty() && (toDateAccepted == null || toDateAccepted.isEmpty()) ) {

            if (!checkDate(fromDateAccepted, simpleDateFormat)){

                //logger.debug("Format from is " + fromDateAccepted != null && !fromDateAccepted.isEmpty());

                throw new IllegalArgumentException("fromDateAccepted date must be formatted as YYYY-MM-DD");

            }

            toDateAccepted = simpleDateFormat.format(new Date());

        }

        if (toDateAccepted != null && !toDateAccepted.isEmpty() && fromDateAccepted != null && !fromDateAccepted.isEmpty()) {

            queryBuilder.append(" and (resultdateofacceptance within \"").append(fromDateAccepted).append(" ").append(toDateAccepted).append("\")");

        }

    }

    public static void enhanceQueryWithYearParams(StringBuilder queryBuilder, HttpServletRequest request) {

        String startYear = request.getParameter("startYear");

        String endYear = request.getParameter("endYear");

        int sYear = -1;

        if (startYear != null && !startYear.isEmpty()) {

            try {

                sYear = Integer.parseInt(startYear);

            } catch(NumberFormatException e) {

                throw new IllegalArgumentException("startYear parameter must be a numeric value.", e);

            }

        }

        int eYear = -1;

        if (endYear !=null && !endYear.isEmpty()) {

            try {

                eYear = Integer.parseInt(endYear);

            } catch(NumberFormatException e) {

                throw new IllegalArgumentException("endYear parameter must be a numeric value.",e);

            }

        }

        if (eYear != -1 && sYear != -1 && eYear < sYear) {

            throw new IllegalArgumentException("endYear must be greater than startYear.");

        }

        addExactQueryTerm("projectstartyear", startYear, queryBuilder);

        addExactQueryTerm("projectendyear", endYear, queryBuilder);

    }

    public static void enhanceQueryWithResultsSortParameters(StringBuilder queryBuilder, HttpServletRequest request) {

        String sortByParameter = request.getParameter("sortBy");

        if (sortByParameter != null) {

            String[] sortParams = sortByParameter.split(",");

            if (sortParams.length != 2) {

                throw new IllegalArgumentException("Invalid sort paremeter. 'sortBy' parameter format is <fieldName>[,ascending|,descending].");

            }

            String sortByField = sortParams[0];

            String order = sortParams[1];

            if (!checkPublicationSortParameterFields(sortByField)){

                throw new IllegalArgumentException("'" + sortByField + "' is not a sortable field.");

            }

            if (!checkOrder(order)) {

                throw new IllegalArgumentException("'" + order + "' is not a valid ordering. Please use one of {ascending, descending}");

            }

            addSortParameter(sortByField, order, queryBuilder);

        }

    }

    public static void enhanceQueryWithProjectSortParameters(StringBuilder queryBuilder, HttpServletRequest request) {

        String sortByParameter = request.getParameter("sortBy");

        if (sortByParameter != null) {

            String[] sortParams = sortByParameter.split(",");

            if (sortParams.length != 2) {

                throw new IllegalArgumentException("Invalid sort paremeter. 'sortBy' parameter format is <fieldName>[,ascending|,descending].");

            }

            String sortByField = sortParams[0];

            String order = sortParams[1];

            if (!checkProjectSortParameterFields(sortByField)){

                throw new IllegalArgumentException("'" + sortByField + "' is not a sortable field.");

            }

            if (!checkOrder(order)) {

                throw new IllegalArgumentException("'" + order + "' is not a valid ordering. Please use one of {ascending, descending}");

            }

            addSortParameter(sortByField, order, queryBuilder);

        }

    }

    private static boolean checkPublicationSortParameterFields(String sortField) {

        if ((sortField != null) && (!sortField.isEmpty()) &&

                sortField.matches("dateofcollection|resultstoragedate|resultembargoenddate|resultembargoendyear|" +

                "resulttypeid|resulttypename|resultlanguageid|resultlanguagename|resultbestlicense|" +

                "resultbestlicenseid|resultdateofacceptance|resultacceptanceyear")) {

            return true;

        }

        return false;

    }

    private static boolean checkProjectSortParameterFields(String sortField) {

        if ((sortField != null) && (!sortField.isEmpty()) &&

                sortField.matches("dateofcollection|projectstartdate|projectstartyear|" +

                        "projectenddate|projectendyear|projectcallidentifier|projectduration|" +

                        "projectecsc39|projectcontracttypeid|projectcontracttypename")) {

            return true;

        }

        return false;

    }

    private static boolean checkOrder(String order) {

        if (order.matches("ascending|descending")) {

            return true;

        }

        return false;

    }

    public static boolean checkDate(String date, SimpleDateFormat simpleDateFormat) {

        try {

            simpleDateFormat.parse(date);

        } catch (ParseException pe) {

            logger.warn("Wrong date format.", pe);

            return false;

        }

        return true;

    }

    /**

     * Enhance the given CQL query with OpenAIRE specific ids

     * @param queryBuilder

     * @param request

     * @return

     */

    public static void enhanceQueryWithOpenAIREIds(StringBuilder queryBuilder, HttpServletRequest request) {

        String[] openairePublicationIDs = request.getParameterValues("openairePublicationID");

        String[] openaireDatasetIDs = request.getParameterValues("openaireDatasetID");

        String[] openaireAuthorIDs = request.getParameterValues("openaireAuthorID");

        String[] openaireProviderIDs  = request.getParameterValues("openaireProviderID");

        String[] openaireProjectIDs  = request.getParameterValues("openaireProjectID");

        enhanceQueryWithIds("objidentifier", openairePublicationIDs, queryBuilder);

        enhanceQueryWithIds("objidentifier", openaireDatasetIDs, queryBuilder);

        enhanceQueryWithIds("relpersonid", openaireAuthorIDs, queryBuilder);

        enhanceQueryWithIds("resulthostingdatasourceid", openaireProviderIDs, queryBuilder);

        enhanceQueryWithIds("relprojectid", openaireProjectIDs, queryBuilder);

    }

    public static void enhanceQueryWithMetadataKeywords(StringBuilder queryBuilder, HttpServletRequest request) {

        String keywords = request.getParameter("keywords");

        String title = request.getParameter("title");

        String author = request.getParameter("author");

        addMetadataQueryTerm(null, keywords, queryBuilder);

        addMetadataQueryTerm("resulttitle", title, queryBuilder);

        addMetadataQueryTerm("relperson", author, queryBuilder);

    }

    public static void enhanceQueryWithProjectMetadataKeywords(StringBuilder queryBuilder, HttpServletRequest request) {

        String keywords = request.getParameter("keywords");

        String acronym = request.getParameter("acronym");

        String name = request.getParameter("name");

        String grantID = request.getParameter("grantID");

        String callID = request.getParameter("callID");

        addMetadataQueryTerm(null, keywords, queryBuilder);

        addMetadataQueryTerm("projectacronym", acronym, queryBuilder);

        addEqualQueryTerm("projecttitle", name, queryBuilder);

        addExactQueryTerm("projectcode", grantID, queryBuilder);

        addExactQueryTerm("projectcallidentifier", callID, queryBuilder);

    }

    public static void enhanceQueryWithParticipantsInfoParams(StringBuilder queryBuilder, HttpServletRequest request) {

        String[] participantCountries = request.getParameterValues("participantCountries");

        String participantAcronyms = request.getParameter("participantAcronyms");

        String openaireParticipantID = request.getParameter("openaireParticipantID");

        if(participantCountries != null) {

            enhanceQueryWithCommaSeparatedValues("relorganizationcountryid", participantCountries, queryBuilder);

        }

        addORQueryTerm("relorganizationname", "relorganizationshortname", participantAcronyms, queryBuilder);

        if(openaireParticipantID != null) {

            addEqualQueryTerm("relorganizationid", openaireParticipantID, queryBuilder);

        }

    }

    public static void enhanceQueryWithSC39Params(StringBuilder queryBuilder, HttpServletRequest request) {

        String sc39 = request.getParameter("sc39");

        addBooleanQueryTerm("projectecsc39", sc39, queryBuilder);

    }

    public static void enhanceQueryWithDoi(StringBuilder queryBuilder, HttpServletRequest request) {

        String[] dois = request.getParameterValues("doi");

        if (dois != null && !(dois.length==0)) {

            queryBuilder.append(" and ");

            for (int i = 0; i < dois.length; i++) {

                String[] commaSeparated = dois[i].split(",");

                for (int j = 0; j < commaSeparated.length; j++) {

                    queryBuilder.append("(pidclassid exact \"doi\" and pid exact \"").append(commaSeparated[j]).append("\")");

                    if (i < dois.length-1 || j < commaSeparated.length-1 ) {

                        queryBuilder.append(" or ");

                    }

                }

            }

        }

    }

    public static void enhanceProjectQueryWithOpenAIREIds(StringBuilder queryBuilder, HttpServletRequest request) {

        String openaireProjectID  = request.getParameter("openaireProjectID");

        if (openaireProjectID != null) {

            addExactQueryTerm("objIdentifier", openaireProjectID, queryBuilder);

        }

    }

    public static void enhanceQueryWithIds(String idIndexFieldName, String[] ids, StringBuilder queryBuilder) {

        if (ids != null && !(ids.length==0)) {

            queryBuilder.append(" and ");

            for (int i = 0; i < ids.length; i++) {

                String[] commaSeparated = ids[i].split(",");

                for (int j = 0; j < commaSeparated.length; j++) {

                    queryBuilder.append("(" + idIndexFieldName + " exact \"").append(commaSeparated[j]).append("\")");

                    if (i < ids.length-1 || j < commaSeparated.length-1 ) {

                        queryBuilder.append(" or ");

                    }

                }

            }

        }

    }

    public static void enhanceQueryWithCommaSeparatedValues(String indexFieldName, String[] fieldValues, StringBuilder queryBuilder) {

        if (fieldValues != null && !(fieldValues.length==0)) {

            queryBuilder.append(" and ");

            for (int i = 0; i < fieldValues.length; i++) {

                String[] commaSeparated = fieldValues[i].split(",");

                for (int j = 0; j < commaSeparated.length; j++) {

                    queryBuilder.append("(").append(indexFieldName).append(" exact \"").append(commaSeparated[j].toUpperCase()).append("\")");

                    if (i < fieldValues.length-1 || j < commaSeparated.length-1 ) {

                        queryBuilder.append(" and ");

                    }

                }

            }

        }

    }

    public static void addMetadataQueryTerm(String indexFieldName, String fieldValue, StringBuilder queryBuilder) {

        if (fieldValue != null && !fieldValue.trim().isEmpty()) {

            if(indexFieldName != null) {

                for (String term: fieldValue.trim().split(" ")){

                    queryBuilder.append(" and (").append(indexFieldName).append(" = ").append(term).append(")");

                }

            } else {

                queryBuilder.append(" and ( " );

                String[] keywords = fieldValue.trim().split(" ");

                for (int i = 0; i < keywords.length; i++) {

                    if (i == keywords.length -1) {

                        queryBuilder.append(keywords[i]);

                    } else {

                        queryBuilder.append(keywords[i]).append(" and ");

                    }

                }

                queryBuilder.append(")" );

            }

        }

    }

    public static void addORQueryTerm(String indexFieldName1, String indexFieldName2, String fieldValue, StringBuilder queryBuilder) {

        if (fieldValue != null && !fieldValue.trim().isEmpty()) {

            for (String term: fieldValue.trim().split(" ")) {

                queryBuilder.append(" and (" + indexFieldName1 + " = " + term + " or  " + indexFieldName2 + " = " + term + ")");

            }

        }

    }

    public static void addORQueryTerm(String[] indexFields, String fieldValue, StringBuilder queryBuilder) {

        if (fieldValue != null && !fieldValue.trim().isEmpty()) {

            queryBuilder.append(" and ( ");

            for (int i = 0; i < indexFields.length; i++) {

                for (String term : fieldValue.trim().split(" ")) {

                    queryBuilder.append(" (" + indexFields[i] + " = " + term + ")");

                }

                if (i == indexFields.length-1) {

                    queryBuilder.append(")");

                } else {

                    queryBuilder.append(" or ");

                }

            }

        }

    }

    private static void addBooleanQueryTerm(String indexFieldName, String requestValue, StringBuilder queryBuilder) {

        if (requestValue != null && !requestValue.trim().isEmpty()) {

            addExactQueryTerm(indexFieldName, requestValue, queryBuilder);

        }

    }

    public static void addBooleanQueryTerm(String indexFieldName, String requestValue, String fieldValue, StringBuilder queryBuilder){

        if (requestValue != null && !requestValue.trim().isEmpty()) {

            if (requestValue.trim().equals("true")) {

                addExactQueryTerm(indexFieldName, fieldValue, queryBuilder);

            } else {

                addDifferentEqualQueryTerm(indexFieldName, fieldValue, queryBuilder);

            }

        }

    }

    public static void addExactQueryTerm(String indexFieldName, String fieldValue, StringBuilder queryBuilder){

        if (fieldValue != null && !fieldValue.trim().isEmpty()) {

            queryBuilder.append(" and (" + indexFieldName + " exact \""+ fieldValue  +"\")");

        }

    }

    public static void addEqualQueryTerm(String indexFieldName, String fieldValue, StringBuilder queryBuilder){

        if (fieldValue != null && !fieldValue.trim().isEmpty()) {

            queryBuilder.append(" and (" + indexFieldName + " = \""+ fieldValue.replaceAll("\"","")  +"\")");

        }

    }

    public static void addNotEqualQueryTerm(String indexFieldName, String fieldValue, StringBuilder queryBuilder){

        if (fieldValue != null && !fieldValue.trim().isEmpty()) {

            queryBuilder.append(" not ").append(indexFieldName).append(" = \"").append(fieldValue).append("\"");

        }

    }

    public static void addDifferentEqualQueryTerm(String indexFieldName, String fieldValue, StringBuilder queryBuilder){

        if (fieldValue != null && !fieldValue.trim().isEmpty()) {

            queryBuilder.append(" and (").append(indexFieldName).append(" <> \"").append(fieldValue).append("\")");

        }

    }

    public static void addVocabularizedQueryTerm(String indexFieldName, String fieldValue, StringBuilder queryBuilder, Vocabulary vocabulary){

        if (fieldValue != null && !fieldValue.trim().isEmpty()) {

            queryBuilder.append(" and (").append(indexFieldName).append(" exact \"").append(devocabularizedTerm(fieldValue, vocabulary)).append("\")");

        }

    }

    private static void addSortParameter(String indexField, String order, StringBuilder queryBuilder) {

        queryBuilder.append(" sortBy " + indexField + "/sort." + order);

    }

    /**

     * Returns the encoding of the given value. If the encoding does not exist

     * in the given vocabulary the method returns the original value.|

     * @param value the value to be encoded

     * @param vocabulary the vocabulary containing the encoding - value mapping

     * @return

     */

    public static String devocabularizedTerm(String value, Vocabulary vocabulary) {

        if (vocabulary != null) {

            String term = vocabulary.getEncoding(value);

            if (term == null) {

                return value;

            }

            return term;

        }

        return value;

    }

}

package eu.dnetlib.data.search.web.api;

import java.util.List;

/**

 * Created by kiatrop on 28/9/2016.

 */

public class CQLQueryGeneration {

    public enum Operator {

        AND("and"),

        OR("or"),

        NOT("not"),

        EQUAL("equal"),

        EXACT("exact");

        private String value;

        Operator(String value) {

            this.value = value;

        }

        private String getValue() {

            return value;

        }

        @Override

        public String toString() {

            return this.getValue();

        }

    }

    public static void appendKeywords(StringBuilder stringBuilder, String keywords) {

        if (keywords != null && !keywords.trim().isEmpty()) {

            stringBuilder.append(" ").append(Operator.AND).append(" (").append(keywords).append(")");

        }

    }

    public static void appendSimpleTerm(StringBuilder stringBuilder, Operator queryOperator, String termValue) {

        stringBuilder.append(' ').append(queryOperator).append(" (").append(termValue).append(')');

    }

    public static void appendSimpleTerms(StringBuilder stringBuilder, Operator queryOperator, List<String> termValues) {

        for (String termValue: termValues) {

            appendSimpleTerm(stringBuilder, queryOperator, termValue);

        }

    }

    public static void appendTerm(StringBuilder stringBuilder, Operator queryOperator, String termName, Operator termOperator, String termValue) {

        stringBuilder.append(' ').append(queryOperator).append(" ( ").

                append(termName).append(' ').append(termOperator).append(' ').append(termValue).append(" )");

    }

    public static void appendTerms(StringBuilder stringBuilder, Operator queryOperator, String termName, Operator termOperator, List<String> termValues) {

        for (String termValue: termValues) {

            appendTerm(stringBuilder, queryOperator, termName, termOperator, termValue);

        }

    }

}

import eu.dnetlib.data.search.web.utils.RequestResponseHandler;

import eu.dnetlib.domain.data.SearchResult;

    @Test

    public void createEntitiesResponseTest(){

        //SearchApiService.getResponseByEntity(String fullQuery, RequestResponseHandler.Entity entity, int offset, int limit, String format, HttpServletRequest request, boolean refine, List<String> refineFields, List<String> fieldQueries, String type) {

        SearchResult emptySearchResult = new SearchResult("oaftype=result", null, 0, 0, 0, null, null, null);

        System.out.println(APIResponseFormatter.createEntitiesResponse(requestMock, RequestResponseHandler.Entity.PUBLICATION, "oaftype=result", null, emptySearchResult, true, MediaType.APPLICATION_JSON));

    }

import java.io.IOException;

import eu.dnetlib.data.search.transform.config.ConfigurationFactory;

public class SolrResultSetOld  { //implements ResultSet<String> {

    /*

/*        TranslatedQuery translatedQuery = translator.getTranslatedQuery(queryParts[0].replace("query=",""));

/*