D-Net

package eu.dnetlib.data.search.solr;

import com.google.gson.Gson;

import eu.dnetlib.api.data.SearchServiceException;

import eu.dnetlib.data.search.transform.Transformer;

import eu.dnetlib.data.search.transform.TransformerException;

import eu.dnetlib.data.search.utils.solr.SolrResultSetOptionsUtil;

import eu.dnetlib.data.search.utils.solr.SolrResultsFormatter;

import eu.dnetlib.domain.EPR;

import gr.uoa.di.driver.enabling.resultset.ResultSet;

import io.micrometer.core.instrument.Timer;

import io.micrometer.prometheus.PrometheusMeterRegistry;

import org.apache.commons.lang.StringEscapeUtils;

import org.apache.log4j.Logger;

import org.apache.solr.client.solrj.SolrClient;

import org.apache.solr.client.solrj.SolrServerException;

import org.apache.solr.client.solrj.impl.CloudSolrClient;

import org.apache.solr.client.solrj.response.FacetField;

import org.apache.solr.client.solrj.response.QueryResponse;

import org.apache.solr.common.SolrDocumentList;

import org.apache.solr.common.params.SolrParams;

import org.apache.solr.common.util.NamedList;

import org.z3950.zing.cql.CQLParseException;

import javax.ws.rs.core.MediaType;

import java.io.IOException;

import java.io.OutputStream;

import java.util.*;

/**

 * Created by antleb on 2/4/14.

 */

public class SolrResultSet implements ResultSet<String> {

    private Logger logger = Logger.getLogger(getClass());

    private EPR epr = null;

    public SolrClient solrClient = null;

    private NamedList<String> queryOpts = new NamedList<String>();

    long size = -1;

    private PrometheusMeterRegistry registry;

    public SolrResultSet(EPR epr, CloudSolrClient cloudSolrClient, PrometheusMeterRegistry registry) throws IOException, CQLParseException {

        logger.debug("Setting solr client " + cloudSolrClient);

        this.epr = epr;

        this.solrClient = cloudSolrClient;

        this.queryOpts = SolrResultSetOptionsUtil.extractQueryOptions(epr.getParameter("query"));

        String layout = epr.getParameter("layout");

        String mdformat = epr.getParameter("mdformat");

        String interpretation = epr.getParameter("interpretation");

        ((CloudSolrClient)solrClient).setDefaultCollection(mdformat + "-" + layout + "-" + interpretation);

        this.registry = registry;

    }

    @Override

    public boolean isOpen() {

        return true;

    }

    @Override

    public boolean isAlive() {

        return true;

    }

    @Override

    public void close() {

/*

        try {

            logger.debug("!!!!!!!!! !!!!!!! CLOSING !!!!!!!!! !!!!!!!!!! ");

            solrClient.close();

        } catch (IOException e) {

            logger.error("Error closing result set.", e);

        }

*/

    }

    @Override

    public int size() {

        return (int) size;

    }

    @Override

    @Deprecated

    public List<String> getElements(int from, int to) {

        return get(from, to);

    }

    List<FacetField> facetFields = null;

    @Override

    @Deprecated

    public List<String> get(int from, int to) {

        List<String> res = new ArrayList<String>();

        QueryResponse rsp = null;

        HashMap<String, List<String>> map = new HashMap<String, List<String>>();

        logger.debug("from: " + from);

        logger.debug("to: " + to);

        queryOpts.add("start", from+1 + "");

        queryOpts.add("rows", to + 1+"");

        try {

            rsp = solrClient.query(SolrParams.toSolrParams(queryOpts));

            facetFields = rsp.getFacetFields();

            SolrDocumentList docs = rsp.getResults();

            if (facetFields!=null && !facetFields.isEmpty()) {

                for (int i = from - 1; i < to; i++) {

                    for (FacetField field : facetFields) {

                        if (field.getValueCount() > i) {

                            BrowseField bf = new BrowseField();

                            bf.setId(field.getValues().get(i).getName());

                            bf.setName(field.getValues().get(i).getName());

                            bf.setCount(field.getValues().get(i).getCount() + "");

                            if (map.get(field.getName()) == null) {

                                map.put(field.getName(), new ArrayList<String>());

                            }

                            map.get(field.getName()).add(new Gson().toJson(bf));

                        }

                    }

                }

                for (Map.Entry<String, List<String>> facetEntry : map.entrySet()) {

                    StringBuilder builder = new StringBuilder();

                    builder.append("\"" + facetEntry.getKey() + "\"" + " : ");

                    builder.append(facetEntry.getValue());

                    res.add(builder.toString());

                }

            }

            logger.debug("time: " + rsp.getElapsedTime());

            logger.debug("found: " + docs.getNumFound());

            logger.debug("docs: " + docs.size());

            for (int i = 0; i < docs.size(); i++) {

                String result = (String) docs.get(i).get("__result");

                res.add(result);

            }

            return res;

        } catch (SolrServerException sse) {

            logger.error("Fail to get results from Solr. ", sse);

        } catch (IOException ioe) {

            logger.error("Fail to get results from Solr. ", ioe);

        }

        return null;

    }

    @Override

    public EPR getEpr() {

        return null;

    }

    public Map<String,List<String>> newGet(int from, int to, String format, Transformer transformer, Transformer oldRefineTransformer) {

        List<String> refineSolrResults = new ArrayList<String>();

        List<String> searchSolrResults = new ArrayList<String>();

        QueryResponse rsp = null;

        HashMap<String, List<String>> map = new HashMap<String, List<String>>();

        queryOpts.add("start", from*to + "");

        queryOpts.add("rows", to +"");

        //queryOpts.add("f.resulthostingdatasource.facet.limit", "2");

        long startTime = System.nanoTime();

        try {

            io.micrometer.core.instrument.Timer.Sample timer = Timer.start(registry);

            rsp = solrClient.query(SolrParams.toSolrParams(queryOpts));

            timer.stop(registry.timer("solr.server.response.duration"));

            long estimatedTime = System.nanoTime() - startTime;

            logger.info("Solrj time " + estimatedTime/1000000 +  " milliseconds for query:" + queryOpts.get("q") +

                    " and facets " + queryOpts.getAll("facet.field") + " and fq " + queryOpts.getAll("fq") + " from: "

                    + from + " and size " + to);

            facetFields = rsp.getFacetFields();

            SolrDocumentList docs = rsp.getResults();

            this.size = docs.getNumFound();

            if (facetFields!=null && !facetFields.isEmpty()) {

                if (format != null && format.equals(MediaType.APPLICATION_JSON)) {

                    for (FacetField field : facetFields) {

                        map.put(field.getName(), new ArrayList<String>());

                        BrowseField bf = null;

                        for (int i = 0; i < field.getValueCount(); i++) {

                            bf = new BrowseField();

                            //bf.setId(org.apache.commons.lang3.StringEscapeUtils.escapeJson(field.getValues().get(i).getName()));

                            bf.setId(field.getValues().get(i).getName());

                            String[] facetedValues = field.getValues().get(i).getName().split("\\|\\|",2);

                            logger.debug("faceted values " + Arrays.toString(facetedValues));

                            if (facetedValues.length > 1) {

                                //bf.setName(org.apache.commons.lang3.StringEscapeUtils.escapeJson(facetedValues[1]));

                                bf.setName(facetedValues[1]);

                                logger.debug("faceted values [1] " + facetedValues[1]);

                            } else if (field.getValues().get(i).getName().split("_\\:\\:",2).length > 1) {

                                //bf.setName(org.apache.commons.lang3.StringEscapeUtils.escapeJson(field.getValues().get(i).getName().split("\\:\\:",2)[1]).replaceAll("\\:\\:", "\\|"));

                                bf.setName(field.getValues().get(i).getName().split("\\:\\:",2)[1].replaceAll("\\:\\:", "\\|"));

                            } else {

                                //bf.setName(org.apache.commons.lang3.StringEscapeUtils.escapeJson(field.getValues().get(i).getName()));

                                bf.setName(field.getValues().get(i).getName());

                            }

                            bf.setCount(field.getValues().get(i).getCount() + "");

                            map.get(field.getName()).add(new Gson().toJson(bf));

                        }

                    }

                    StringBuilder builder = null;

                    for (Map.Entry<String, List<String>> facetEntry : map.entrySet()) {

                        builder = new StringBuilder();

                        builder.append("\"" + facetEntry.getKey() + "\"" + " : ");

                        builder.append(facetEntry.getValue());

                        refineSolrResults.add(builder.toString());

                    }

                } else { //the old implementation & xml as default

                    logger.debug("Creating old browse results.");

                    createXmlRefineFields(refineSolrResults, oldRefineTransformer);

                }

            }

            for (int i = 0; i < docs.size(); i++) {

                String result = (String) docs.get(i).get("__result");

                logger.debug("["+ i +"]: " + docs.get(i).get("__result"));

                try {

                    if (transformer != null) {

                        logger.debug("1 >>>>>>" + result);

                        String xml = result.replaceAll("<em>","").replaceAll("</em>","");

                        result = transformer.transform(xml);

                        logger.debug("2 >>>>>>" + result);

                    }

                } catch (TransformerException te) {

                    logger.warn("Error transforming " + result, te);

                    continue;

                }

                if (format != null && format.equals(MediaType.APPLICATION_JSON)) {

                    searchSolrResults.add(SolrResultsFormatter.xml2Json(result));

                } else { // default xml

                    searchSolrResults.add(result);

                }

            }

            Map<String,List<String>> response = new HashMap<String, List<String>>();

            //logger.debug("refine results " + refineSolrResults);

            //logger.info("search results SIZE " + searchSolrResults.size());

            //logger.info("search results " + searchSolrResults);

            response.put("refine",refineSolrResults);

            response.put("search", searchSolrResults);

            return response;

        } catch (SolrServerException sse) {

            logger.error("Error calling Solr.", sse);

        } catch (IOException ioe) {

            logger.error("Error calling Solr.", ioe);

        }

        return null;

    }

    /**

     * limit is the maximum number of results the cursor get is allowed to fetch. If limit is set to -1 all

     * results are returned.

     */

    public void cursorGet(Transformer transformer, int limit, OutputStream os) throws SolrServerException, SearchServiceException {

        int rows = 500;

        int limitCounter = -1;

        queryOpts.add("start", "0");

        queryOpts.add("rows", "0");

        queryOpts.remove("rows");

        queryOpts.add("rows", rows+"");

        queryOpts.add("fl", "__result");

        queryOpts.add("shards.tolerant","true");

        queryOpts.add("cursorMark", "*");

        queryOpts.add("sort", "__indexrecordidentifier asc");

        String cursorMark = "*";

        String nextCursorMark = "";

        if ( limit > 0 ) {

            limitCounter = limit/rows;

            logger.info("limit counter " + limitCounter);

        }

        try {

            QueryResponse resp = solrClient.query(SolrParams.toSolrParams(queryOpts));

            while (!cursorMark.equals(nextCursorMark) && ( limitCounter > 0 || limitCounter == -1)) {

                //logger.info(">> " + limitCounter);

                resp = solrClient.query(SolrParams.toSolrParams(queryOpts));

                cursorMark = nextCursorMark;

                nextCursorMark = resp.getNextCursorMark();

                for (int i = 0; i < resp.getResults().size(); i++) {

                    if (transformer != null) {

                        String result = null;

                        try {

                            logger.debug("PRE RESULT " + resp.getResults().get(i).get("__result"));

                            result = transformer.transform((String) resp.getResults().get(i).get("__result"));

                            logger.debug("RESULT " + result);

                        } catch (TransformerException te) {

                            logger.warn("Error transforming " + result, te);

                            continue;

                        }

                        try {

                            os.write(result.getBytes());

                            os.flush();

                        } catch (IOException e) {

                            logger.error("Cursor get... ", e);

                            continue;

                        }

                    }

                }

                queryOpts.remove("cursorMark");

                queryOpts.add("cursorMark", nextCursorMark);

                limitCounter--;

            }

        } catch (IOException ioe) {

            logger.error("Error executing solr query. ", ioe);

        }

    }

    //TODO get rid of this as soon as Joomla portal is out

    //Just copied and refactored the old one...

    @Deprecated

    private void createXmlRefineFields(List<String> res, Transformer oldRefineTransformer) {

        int max = -12;

        for (FacetField field:facetFields) {

            logger.debug("field " + field.getName() + " has count " + field.getValueCount());

            if (field.getValueCount() > max) {

                max = field.getValueCount();

            }

        }

        logger.debug("max " + max);

        for (int i = 0; i < max; i++) {

            StringBuilder sb = new StringBuilder();

            sb.append("<row>");

            for (FacetField field:facetFields) {

                if (field.getValueCount() > i) {

                    sb.append("<groupresult field=\"").append(field.getName()).append("\">");

                    sb.append("<count>").append(field.getValues().get(i).getCount()).append("</count>");

                    sb.append("<originalValue>").append(StringEscapeUtils.escapeXml(field.getValues().get(i).getName())).append("</originalValue>");

                    String[] facetValues = field.getValues().get(i).getName().split("\\|\\|");

                    if(facetValues.length > 1) {

                        sb.append("<value>").append(StringEscapeUtils.escapeXml(facetValues[1])).append("</value>");

                    } else {

                        sb.append("<value>").append(StringEscapeUtils.escapeXml(facetValues[0])).append("</value>");

                    }

                    sb.append("</groupresult>");

                }

            }

            sb.append("</row>");

            try {

                //logger.debug("row: " + sb.toString());

                //logger.debug("row2: " + oldRefineTransformer.transform(sb.toString()));

                //TODO remove

                res.add(oldRefineTransformer.transform(sb.toString()));

            } catch (TransformerException te) {

                logger.error("Cannot transform refine for: " + sb.toString(), te);

            }

        }

    }

 /*   public static void main(String[] args) throws IOException, CQLParseException, SolrServerException {

        CloudSolrServer solrClient = new CloudSolrServer("beta.solr.openaire.eu:9983");

        solrClient.setDefaultCollection("DMF-index-openaire");

        NamedList<String> queryOpts = new NamedList<String>();

        //q=*:*&start=0&rows=10&cursorMark=*&sort=dateofcollection asc

        queryOpts.add("q", new CqlTranslatorImpl().getTranslatedQuery("objIdentifier = acm_________::0002c24f82c295e925a2bdf7bbf49bfc").asLucene());

        queryOpts.add("start", "0");

        queryOpts.add("rows", "1");

        queryOpts.add("fl", "__result");

        queryOpts.add("shards.tolerant","true");

        queryOpts.add("cursorMark", "*");

        queryOpts.add("sort", "__indexrecordidentifier asc");

        //queryOpts.add("q", new CqlTranslatorImpl().getTranslatedQuery("oaftype exact project").asLucene());

        NamedList<String> extraOpts = new NamedList<String>();

        QueryResponse resp = solrClient.query(SolrParams.toSolrParams(queryOpts));

        System.out.println("results " + resp.getResults().size());

        String cursorMark = "*";

        String nextCursorMark = "";

        int curs = 0;

        while (!cursorMark.equals(nextCursorMark)) {

            System.out.println("cursor " + cursorMark);

            System.out.println("next cursor " + nextCursorMark);

            cursorMark = nextCursorMark;

            for (int i = 0; i < resp.getResults().size(); i++) {

                String result = ((ArrayList<String>) resp.getResults().get(i).get("__result")).get(0);

                //System.out.println(result);

                resp = solrClient.query(SolrParams.toSolrParams(queryOpts));

            }

            nextCursorMark = resp.getNextCursorMark();

            queryOpts.add("cursorMark", nextCursorMark);

            System.out.println("CURS " + curs);

            curs ++;

        }

        //System.out.println((new CqlTranslatorImpl().getTranslatedQuery("objIdentifier = acm_________::0002c24f82c295e925a2bdf7bbf49bfc").asLucene()));

        //extraOpts.add("start", "1");

       // extraOpts.add("rows", "10");

       // extraOpts.addAll(queryOpts);

        //queryOpts.add("facet", "true");

        //TranslatedQuery translatedQuery = new CqlTranslatorImpl().getTranslatedQuery("oaftype=result sortBy resultdateofacceptance/sort.descending");

     //   queryOpts.add("q", "oaftype=project");

        //queryOpts.add("facet", "true");

        //queryOpts.add("facet.mincount", "1");

        //queryOpts.add("fq", "popularity");

//        queryOpts.put("fq", new CqlTranslatorImpl().getTranslatedQuery("").asLucene());

       // queryOpts.add("facet.field", "contextid");

       //  queryOpts.add("facet.field", "contextname");

       //  queryOpts.add("facet.mincount", "1");

       //  queryOpts.add("facet.threads", "10");

       // System.out.println(translatedQuery.getOptions().getSort().getMode());

       // System.out.println(translatedQuery.getOptions().getSort().getField());

        //queryOpts.add("sort", translatedQuery.getOptions().getSort().getField() + " " + translatedQuery.getOptions().getSort().getMode() );

/*        QueryResponse resp = null;

        synchronized (solrClient) {

            resp = solrClient.query(SolrParams.toSolrParams(extraOpts));

        }*/

//        System.out.println("time: " + resp.getElapsedTime());

    //System.out.println("results: " + resp.getResults());

/*      System.out.println(resp.getFacetField("contextname").getValueCount());

        for (FacetField.Count count:resp.getFacetField("contextname").getValues())

            System.out.println(count.getName() + " : " +  count.getCount());

        int max = -12;

        for (FacetField field:resp.getFacetFields()) {

            if (field.getValueCount() > max)

                max = field.getValueCount();

        }

        System.out.println("max: " + max);

*/

 //   }

//    @Override

//    public EPR getEpr() {

//        return epr;

//   }

}

class BrowseField {

    String name;

    public String getName() {

        return name;

    }

    public void setName(String name) {

        this.name = name;

    }

    public String getId() {

        return id;

    }

    public void setId(String id) {

        this.id = id;

    }

    public String getCount() {

        return count;

    }

    public void setCount(String count) {

        this.count = count;

    }

    String id;

    String count;

}