D-Net

package eu.dnetlib.data.search.app;

import eu.dnetlib.api.data.IndexService;

import eu.dnetlib.api.data.IndexServiceException;

import eu.dnetlib.api.data.SearchService;

import eu.dnetlib.api.data.SearchServiceException;

import eu.dnetlib.api.enabling.ISLookUpService;

import eu.dnetlib.common.rmi.UnimplementedException;

import eu.dnetlib.data.search.app.plan.FieldRewriteRule;

import eu.dnetlib.data.search.app.plan.QueryRewriteRule;

import eu.dnetlib.data.search.solr.SolrResultSet;

import eu.dnetlib.data.search.transform.Transformer;

import eu.dnetlib.data.search.transform.config.SearchRegistry;

import eu.dnetlib.data.search.transform.formatter.Formatter;

import eu.dnetlib.domain.ActionType;

import eu.dnetlib.domain.EPR;

import eu.dnetlib.domain.ResourceType;

import eu.dnetlib.domain.data.FormattedSearchResult;

import eu.dnetlib.domain.data.SearchResult;

import eu.dnetlib.domain.data.SuggestiveResult;

import eu.dnetlib.domain.enabling.Notification;

import gr.uoa.di.driver.app.DriverServiceImpl;

import gr.uoa.di.driver.enabling.issn.NotificationListener;

import gr.uoa.di.driver.enabling.resultset.ResultSet;

import gr.uoa.di.driver.enabling.resultset.ResultSetFactory;

import gr.uoa.di.driver.util.ServiceLocator;

import io.micrometer.core.instrument.Timer;

import io.micrometer.prometheus.PrometheusMeterRegistry;

import org.apache.log4j.Logger;

import org.apache.solr.client.solrj.SolrServerException;

import org.springframework.beans.factory.annotation.Autowired;

import org.w3c.dom.Document;

import org.w3c.dom.Node;

import org.xml.sax.InputSource;

import javax.xml.parsers.DocumentBuilder;

import javax.xml.parsers.DocumentBuilderFactory;

import javax.xml.xpath.XPath;

import javax.xml.xpath.XPathConstants;

import javax.xml.xpath.XPathExpression;

import javax.xml.xpath.XPathFactory;

import java.io.OutputStream;

import java.io.StringReader;

import java.util.*;

//import eu.dnetlib.utils.cql.CqlException;

public class SearchServiceImpl extends DriverServiceImpl

        implements SearchService {

    private static Logger logger = Logger.getLogger(SearchServiceImpl.class);

    //@Deprecated

    //private static Logger tlogger = Logger.getLogger("eu.dnetlib.data.search.app.Timer");

    private String mdFormat = "DMF";

    private String indexLayout = "index";

    private ServiceLocator<IndexService> indexLocator = null;

    private ServiceLocator<ISLookUpService> lookUpServiceServiceLocator = null;

    private ResultSetFactory rsFactory = null;

    private SearchRegistry transformerFactory = null;

    private List<QueryRewriteRule> queryRules = null;

    private List<String> fieldQueryRules = null;

    private Map<String, FieldRewriteRule> fieldRules = null;

    private boolean enableBrowseCache = false;

    private SearchServiceBlackboardHandler blackboardNotificationHandler = null;

    @Autowired

    private PrometheusMeterRegistry registry;

    //private CQLParser cqlParser = null;

    @Override

    public void init() {

        super.init();

        String serviceId = this.getServiceEPR().getParameter("serviceId");

        this.subscribe(

                ActionType.UPDATE,

                ResourceType.ANY.SEARCHSERVICERESOURCETYPE,

                serviceId,

                "RESOURCE_PROFILE/BODY/BLACKBOARD/LAST_REQUEST",

                new NotificationListener() {

                    @Override

                    public void processNotification(Notification notification) {

                        blackboardNotificationHandler.notified(

                                notification.getSubscriptionId(),

                                notification.getTopic(),

                                notification.getIsId(),

                                notification.getMessage());

                    }

                });

        try {

            String searchProfile = lookUpServiceServiceLocator.getService().getResourceProfile(serviceId);

            if (searchProfile != null) {

                DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();

                dbf.setNamespaceAware(true);

                DocumentBuilder db = dbf.newDocumentBuilder();

                Document doc = db.parse(new InputSource(new StringReader(searchProfile)));

                XPathFactory factory = XPathFactory.newInstance();

                XPath xpath = factory.newXPath();

                XPathExpression searchMdFormatExpression = xpath.compile("//SERVICE_PROPERTIES/PROPERTY[@key='mdformat']");

                Node node = (Node) searchMdFormatExpression.evaluate(doc,XPathConstants.NODE);

                if (node != null){

                    String profileMdFormat = node.getAttributes().getNamedItem("value").getTextContent();

                    if (profileMdFormat != null) {

                        //logger.debug("mdformat in properties " + mdFormat );

                        logger.info("Setting mdformat to '" + profileMdFormat + "'");

                        mdFormat = profileMdFormat;

                    }

                }

            }

        } catch (Exception e) {

            logger.error("Fail to load search service profile with id " + serviceId + " from IS.", e);

        }

    }

    @Override

    public SuggestiveResult suggestiveSearch(String query) throws SearchServiceException {

        throw new UnimplementedException();

    }

    @Override

    @Deprecated

    public SearchResult search(String text, String transformer, String locale, int page, int size) throws SearchServiceException {

        return searchNrefine(text, transformer, null, locale, page, size, null);

    }

    @Override

    @Deprecated

    public SearchResult refine(String text, String transformer, String locale, Collection<String> fields) throws SearchServiceException {

        return searchNrefine(text, null, transformer, locale, 1, -1, fields);

    }

    @Override

    @Deprecated

    public SearchResult searchNrefine(String text, String searchTransformer, String browseTransformer,

                                      String locale, int page, int size, Collection<String> fields) throws SearchServiceException {

        //logger.info("deprecated searchNrefine > from: " + page + " to:" + size);

        //TODO check locale

        //logger.debug("Search transformer " + searchTransformer);

        Transformer sTransformer = transformerFactory.getTransformer(searchTransformer, Locale.getDefault());

        Transformer oldRefineTransformer = transformerFactory.getTransformer("results_openaire_browse", Locale.getDefault());

        //logger.debug("Refine transformer " + browseTransformer);

        //Transformer rTranformer = transformerFactory.getTransformer(refineTransformer, Locale.getDefault());

        List<String> refineFields = null;

        if (fields!=null) {

            refineFields = new ArrayList<String>(fields);

        }

        return newSearch(text, locale, refineFields, null, new ArrayList<String>(), page, size, "", sTransformer, oldRefineTransformer, true);

    }

    @Override

    public FormattedSearchResult search(String queryText, String transformerName, String format, String locale, int page, int size)

            throws SearchServiceException {

        return searchNrefine(queryText, transformerName, null, format, locale, page, size, null);

    }

    @Override

    public FormattedSearchResult refine(String queryText, String refineTransformer, String format, String locale, Collection<String> fields) throws SearchServiceException {

        return searchNrefine(queryText, null, refineTransformer, format, locale, 0, -1, fields);

    }

    @Override

    public FormattedSearchResult searchNrefine(String queryText,

                                               String searchTransformer, String refineTransformer, String format,

                                               String locale, int page, int size, Collection<String> fields)  throws SearchServiceException {

        //logger.info("searchNrefine > from: " + page + " to:" + size);

        //TODO check locale

        FormattedSearchResult formattedSearchResult = null;

        //logger.debug("Search transformer " + searchTransformer);

        Transformer sTransformer = transformerFactory.getTransformer(searchTransformer, Locale.getDefault());

        Transformer oldRefineTransformer = transformerFactory.getTransformer("results_openaire_browse", Locale.getDefault());

        //logger.debug("Refine transformer " + refineTransformer);

        //Transformer rTranformer = transformerFactory.getTransformer(refineTransformer, Locale.getDefault());

        List<String> refineFields = null;

        if (fields!=null) {

            refineFields = new ArrayList<String>(fields);

        }

        SearchResult searchResult = newSearch(queryText, locale, refineFields, new ArrayList<>(), new ArrayList<String>(), page, size, format, sTransformer, oldRefineTransformer, true);

        Formatter formatter = transformerFactory.getFormatter(format); // formatter cannot be returned as null

        try {

            formattedSearchResult = new FormattedSearchResult(formatter.format(searchResult), searchResult.getTotal());

        } catch (Exception e) {

            logger.error("Error formating search results.", e);

        }

        return formattedSearchResult;

    }

    public SearchResult newSearch (String text, String locale, List<String> refinefields, List<String> specialFacets, List<String> fieldQueries,

                                   int from, int to, String format, Transformer transformer, Transformer oldRefineTransformer,

                                   boolean oldPaging) throws SearchServiceException {

        logger.info("newSearch > from: " + from + " to:" + to);

        long startTime = System.nanoTime();

        Timer.Sample timer = Timer.start(registry);

        IndexService index = getIndexLocator().getService();

        EPR epr = null;

        ResultSet<String> rs = null;

        List<String> browseResults = null;

        List<String> searchResults = null;

        String query = rewrite(text);

        enhanceFieldQueries(fieldQueries);

        logger.info("Performing query '" + query + "' and fields " + fieldQueries + " and refine " + refinefields);

        try {

            //TODO see parser and maybe delete!

            //query = new CQLParser().parse(query).toCQL();

            String eprQuery = createEprQuery(query, refinefields, specialFacets, fieldQueries);

            epr = index.getBrowsingStatistics(eprQuery, "all", mdFormat, indexLayout);

            if (epr == null){

                throw new SearchServiceException("Something really strange happened there! Index returned null result set id.");

            }

            //get the locale TODO do we need this?

            //String correctLocale = getCorrectLocale(locale);

            //StringTokenizer tokenizer = new StringTokenizer(correctLocale, "_");

            //Locale requestLocale = new Locale(tokenizer.nextToken(), tokenizer.nextToken());

            rs = rsFactory.createResultSet(epr);

            Map<String, List<String>> list = null;

            if (oldPaging) {

                list = ((SolrResultSet)rs).newGet(from-1, to, format, transformer, oldRefineTransformer);

            } else {

                list = ((SolrResultSet)rs).newGet(from, to, format, transformer, oldRefineTransformer);

            }

            searchResults = list.get("search");

            browseResults = list.get("refine");

        } catch (IndexServiceException ise) {

            logger.error("Error getting refine results.", ise);

            throw new SearchServiceException("Error getting refine results.", ise);

        } finally {

            timer.stop(registry.timer("search.server.response.duration"));

/*

            histogram.observe(System.currentTimeMillis()-startTime);

*/

        }

        long estimatedTime = System.nanoTime() - startTime;

        logger.debug("Search time " + estimatedTime/1000000 +  " milliseconds for query " + query +

                " and fields " + fieldQueries + " and refine " + refinefields + " from: "+ from + " and size " + to);

        //logger.info("Returned results for NEW search query '" + query + "' and fields " + fieldQueries + " and refine " + refinefields);;

        rs.close();

        return new SearchResult(query, Locale.getDefault().toString(), rs.size(), from, to, searchResults, browseResults, refinefields);

    }

    public void cursorSearch(String text, List<String> refinefields, List<String> specialFacets, List<String> fieldQueries,

                             String format, Transformer transformer, OutputStream os) throws SearchServiceException {

        long startTime = System.nanoTime();

        IndexService index = getIndexLocator().getService();

        EPR epr = null;

        ResultSet<String> rs = null;

        String query = rewrite(text);

        enhanceFieldQueries(fieldQueries);

        logger.info("Performing cursor query " + query + "' and fields " + fieldQueries + " and refine " + refinefields);

        logger.debug("Performing cursor query " + query + "' and fields " + fieldQueries + " and refine " + refinefields);

        try {

            String eprQuery = createEprQuery(query, refinefields, specialFacets, fieldQueries);

            epr = index.getBrowsingStatistics(eprQuery, "all", mdFormat, indexLayout);

            if (epr == null) {

                throw new SearchServiceException("Something really strange happened there! Index returned null result set id.");

            }

            rs = rsFactory.createResultSet(epr);

            ((SolrResultSet)rs).cursorGet(transformer,2000, os);

        } catch (IndexServiceException ise) {

            logger.error("Error getting cursor results.", ise);

            throw new SearchServiceException("Error getting cursor results.", ise);

        } catch (SolrServerException sse) {

            logger.error("Error getting cursor results.", sse);

            throw new SearchServiceException("Error getting cursor results.", sse);

        }

        long estimatedTime = System.nanoTime() - startTime;

        logger.debug("Cursor search time " + estimatedTime/1000000 +  " milliseconds for query " + query +

                " and fields " + fieldQueries + " and refine " + refinefields);

        rs.close();

    }

    private String rewrite(String query) {

        if (queryRules != null) {

            for (QueryRewriteRule queryRule: queryRules) {

                if (logger.isDebugEnabled()) {

                    logger.debug("Apply rule " + query);

                }

                query = queryRule.apply(query);

                if (logger.isDebugEnabled()) {

                    logger.debug("Rewritten query is " + query);

                }

            }

        }

        return query;

    }

    private void enhanceFieldQueries(List<String> fieldQueries) {

        if (fieldQueries != null && fieldQueryRules != null && !fieldQueryRules.isEmpty()) {

            fieldQueries.addAll(fieldQueryRules);

        }

    }

    public void setFieldRules(Collection<FieldRewriteRule> fieldRules) {

        this.fieldRules = new HashMap<String, FieldRewriteRule>();

        for (FieldRewriteRule rule : fieldRules) {

            String key = rule.getFieldName();

            if (this.fieldRules.containsKey(key)) {

                logger.warn("Multiple rules for field " + key);

                logger.warn("Keeping last rule " + rule.getName());

            }

            this.fieldRules.put(key, rule);

        }

    }

    public static String createEprQuery(String query, List<String> refineFields, List<String> specialFacets, List<String> fieldQueries) {

        StringBuffer queryBuffer = new StringBuffer();

        queryBuffer.append("query=");

        StringBuffer facetsBuffer = new StringBuffer();

        facetsBuffer.append("&groupby=");

        StringBuffer fqBuffer = new StringBuffer();

        fqBuffer.append("&fq=");

        StringBuffer sfBuffer = new StringBuffer();

        sfBuffer.append("&sf=");

        if (query != null) { //TODO consider exception?

            queryBuffer.append(query);

        }

        if(refineFields != null) {

            for (Iterator<String> iterator = refineFields.iterator(); iterator.hasNext(); ) {

                facetsBuffer.append(iterator.next());

                if (iterator.hasNext()) {

                    facetsBuffer.append(",");

                }

            }

        }

        if(specialFacets != null) {

            for (Iterator<String> iterator = specialFacets.iterator(); iterator.hasNext(); ) {

                sfBuffer.append(iterator.next());

                if (iterator.hasNext()) {

                    sfBuffer.append(",");

                }

            }

        }

       // logger.debug("special buffer " + sfBuffer.toString());

        if(fieldQueries != null) {

            for (Iterator<String> iterator = fieldQueries.iterator(); iterator.hasNext(); ) {

                fqBuffer.append(iterator.next());

                if (iterator.hasNext()) {

                    fqBuffer.append(",");

                }

            }

        }

        return queryBuffer.append(facetsBuffer.toString()).append(sfBuffer.toString()).append(fqBuffer.toString()).toString();

    }

    //TODO: I wish to remove this. This was only made (quick and dirty - only the enhanceFieldQueries(fieldQueries) is missing

    //from newSearch() after a last time request for the portal to show all the publications and the deletedbyinference ones.

    //I did not want to pass a parameter since I do not know if we are going to keep it. This is for a tech meeting showcase.

    //If we want to keep this I need to redesign.

    public SearchResult newSearchWithoutFieldQueries (String text, String locale, List<String> refinefields, List<String> specialFacets,

                                                      List<String> fieldQueries, int from, int to, String format, Transformer transformer,

                                                      Transformer oldRefineTransformer, boolean oldPaging) throws SearchServiceException {

        logger.info("non filtered search for...  > from: " + from + " to:" + to);

        long startTime = System.nanoTime();

        IndexService index = getIndexLocator().getService();

        EPR epr = null;

        ResultSet<String> rs = null;

        List<String> browseResults = null;

        List<String> searchResults = null;

        String query = rewrite(text);

        logger.info("Performing query '" + query + "' and fields " + fieldQueries + " and refine " + refinefields);

        try {

            //TODO see parser and maybe delete!

            //query = new CQLParser().parse(query).toCQL();

            String eprQuery = createEprQuery(query, refinefields, specialFacets, fieldQueries);

            epr = index.getBrowsingStatistics(eprQuery, "all", mdFormat, indexLayout);

            if (epr == null) {

                throw new SearchServiceException("Something really strange happened there! Index returned null result set id.");

            }

            //get the locale TODO do we need this?

            //String correctLocale = getCorrectLocale(locale);

            //StringTokenizer tokenizer = new StringTokenizer(correctLocale, "_");

            //Locale requestLocale = new Locale(tokenizer.nextToken(), tokenizer.nextToken());

            rs = rsFactory.createResultSet(epr);

            Map<String, List<String>> list = null;

            if (oldPaging) {

                list = ((SolrResultSet)rs).newGet(from-1, to, format, transformer, oldRefineTransformer);

            } else {

                list = ((SolrResultSet)rs).newGet(from, to, format, transformer, oldRefineTransformer);

            }

            searchResults = list.get("search");

            browseResults = list.get("refine");

        } catch (IndexServiceException ise) {

            logger.error("Error getting refine results.", ise);

            throw new SearchServiceException("Error getting refine results.", ise);

        }

        long estimatedTime = System.nanoTime() - startTime;

        logger.debug("Search time " + estimatedTime/1000000 +  " milliseconds for query " + query +

                " and fields " + fieldQueries + " and refine " + refinefields + " from: "+ from + " and size " + to);

        //logger.info("Returned results for NEW search query '" + query + "' and fields " + fieldQueries + " and refine " + refinefields);;

        rs.close();

        return new SearchResult(query, Locale.getDefault().toString(), rs.size(), from, to, searchResults, browseResults, refinefields);

    }

    public String getMdFormat() {

        return mdFormat;

    }

    public void setMdFormat(String mdFormat) {

        this.mdFormat = mdFormat;

    }

    public ServiceLocator<IndexService> getIndexLocator() {

        return indexLocator;

    }

    public void setIndexLocator(ServiceLocator<IndexService> indexLocator) {

        this.indexLocator = indexLocator;

    }

    public ResultSetFactory getRsFactory() {

        return rsFactory;

    }

    public void setRsFactory(ResultSetFactory rsFactory) {

        this.rsFactory = rsFactory;

    }

    public Collection<FieldRewriteRule> getFieldRules() {

        return fieldRules.values();

    }

    public List<QueryRewriteRule> getQueryRules() {

        return queryRules;

    }

    public void setQueryRules(List<QueryRewriteRule> queryRules) {

        this.queryRules = queryRules;

    }

    public boolean isEnableBrowseCache() {

        return enableBrowseCache;

    }

    public void setEnableBrowseCache(boolean enableBrowseCache) {

        this.enableBrowseCache = enableBrowseCache;

    }

    public SearchRegistry getTransformerFactory() {

        return transformerFactory;

    }

    public void setTransformerFactory(SearchRegistry transformerFactory) { this.transformerFactory = transformerFactory; }

    public String getIndexLayout() {

        return indexLayout;

    }

    public void setIndexLayout(String indexLayout) {

        this.indexLayout = indexLayout;

    }

    public SearchServiceBlackboardHandler getBlackboardNotificationHandler() {

        return blackboardNotificationHandler;

    }

    public void setBlackboardNotificationHandler(SearchServiceBlackboardHandler blackboardNotificationHandler) {

        this.blackboardNotificationHandler = blackboardNotificationHandler;

    }

    public void setLookUpServiceServiceLocator(ServiceLocator<ISLookUpService> lookUpServiceServiceLocator) {

        this.lookUpServiceServiceLocator = lookUpServiceServiceLocator;

    }

    public List<String> getFieldQueryRules() {

        return fieldQueryRules;

    }

    public void setFieldQueryRules(List<String> fieldQueryRules) {

        this.fieldQueryRules = fieldQueryRules;

    }

}