D-Net

package eu.dnetlib.data.search.app;

import eu.dnetlib.api.data.IndexService;

import eu.dnetlib.api.data.IndexServiceException;

import eu.dnetlib.api.data.SearchService;

import eu.dnetlib.api.data.SearchServiceException;

import eu.dnetlib.api.enabling.ISLookUpService;

import eu.dnetlib.common.rmi.UnimplementedException;

import eu.dnetlib.data.search.app.plan.FieldRewriteRule;

import eu.dnetlib.data.search.app.plan.QueryRewriteRule;

import eu.dnetlib.data.search.transform.Transformer;

import eu.dnetlib.data.search.transform.TransformerException;

import eu.dnetlib.data.search.transform.config.SearchRegistry;

import eu.dnetlib.data.search.transform.formatter.Formatter;

import eu.dnetlib.domain.ActionType;

import eu.dnetlib.domain.EPR;

import eu.dnetlib.domain.ResourceType;

import eu.dnetlib.domain.data.FormattedSearchResult;

import eu.dnetlib.domain.data.SearchResult;

import eu.dnetlib.domain.data.SuggestiveResult;

import eu.dnetlib.domain.enabling.Notification;

import gr.uoa.di.driver.app.DriverServiceImpl;

import gr.uoa.di.driver.enabling.issn.NotificationListener;

import gr.uoa.di.driver.enabling.resultset.ResultSet;

import gr.uoa.di.driver.enabling.resultset.ResultSetFactory;

import gr.uoa.di.driver.util.ServiceLocator;

import org.apache.log4j.Logger;

import org.w3c.dom.Document;

import org.w3c.dom.Node;

import org.xml.sax.InputSource;

import org.z3950.zing.cql.CQLParseException;

import org.z3950.zing.cql.CQLParser;

import javax.xml.parsers.DocumentBuilder;

import javax.xml.parsers.DocumentBuilderFactory;

import javax.xml.xpath.XPath;

import javax.xml.xpath.XPathConstants;

import javax.xml.xpath.XPathExpression;

import javax.xml.xpath.XPathFactory;

import java.io.IOException;

import java.io.StringReader;

import java.util.*;

//import eu.dnetlib.utils.cql.CqlException;

public class SearchServiceImpl extends DriverServiceImpl

        implements SearchService {

    private static Logger logger = Logger.getLogger(SearchServiceImpl.class);

    @Deprecated

    private static Logger tlogger = Logger.getLogger("eu.dnetlib.data.search.app.Timer");

    private String mdFormat = "DMF";

    private String indexLayout = "index";

    private ServiceLocator<IndexService> indexLocator = null;

    private ServiceLocator<ISLookUpService> lookUpServiceServiceLocator = null;

    private ResultSetFactory rsFactory = null;

    private SearchRegistry transformerFactory = null;

    private List<QueryRewriteRule> queryRules = null;

    private Map<String, FieldRewriteRule> fieldRules = null;

    private boolean enableBrowseCache = false;

    private SearchServiceBlackboardHandler blackboardNotificationHandler = null;

    private CQLParser cqlParser = null;

    @Override

    public void init() {

        super.init();

        String serviceId = this.getServiceEPR().getParameter("serviceId");

        this.subscribe(

                ActionType.UPDATE,

                ResourceType.SEARCHSERVICERESOURCETYPE,

                serviceId,

                "RESOURCE_PROFILE/BODY/BLACKBOARD/LAST_REQUEST",

                new NotificationListener() {

                    @Override

                    public void processNotification(Notification notification) {

                        blackboardNotificationHandler.notified(

                                notification.getSubscriptionId(),

                                notification.getTopic(),

                                notification.getIsId(),

                                notification.getMessage());

                    }

                });

        try {

            String searchProfile = lookUpServiceServiceLocator.getService().getResourceProfile(serviceId);

            if (searchProfile != null) {

                DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();

                dbf.setNamespaceAware(true);

                DocumentBuilder db = dbf.newDocumentBuilder();

                Document doc = db.parse(new InputSource(new StringReader(searchProfile)));

                XPathFactory factory = XPathFactory.newInstance();

                XPath xpath = factory.newXPath();

                XPathExpression searchMdFormatExpression = xpath.compile("//SERVICE_PROPERTIES/PROPERTY[@key='mdformat']");

                Node node = (Node) searchMdFormatExpression.evaluate(doc,XPathConstants.NODE);

                if (node != null){

                    String profileMdFormat = node.getAttributes().getNamedItem("value").getTextContent();

                    if (profileMdFormat != null) {

                        logger.debug("mdformat in properties " + mdFormat );

                        logger.info("Setting mdformat to '" + profileMdFormat + "'");

                        mdFormat = profileMdFormat;

                    }

                }

            }

        } catch (Exception e) {

            logger.error("Fail to load search service profile with id " + serviceId + " from IS.", e);

        }

    }

    @Override

    public SuggestiveResult suggestiveSearch(String query)

            throws SearchServiceException {

        throw new UnimplementedException();

	/*	logger.debug("running suggestive search for " + query);

		SuggestiveResult suggestiveResult = new SuggestiveResult();

		IndexService index = getIndexLocator().getService();

		try {

			Hint hint = index.suggestiveSearch("all", "query=" + text, mdFormat, "index", "SimpleRatioHeuristics");

			String alternateTerm = hint.getAlternateTerm();

			boolean autofollow = hint.isAutoFollowHint();

			logger.debug("alternateTerm " + alternateTerm + " autofollow " + autofollow);

			if (alternateTerm != null) {

				suggestiveResult.setAlternativeTerm(alternateTerm);

				suggestiveResult.setAutofollow(autofollow);

				if (autofollow) {

					logger.debug("getting epr for " + alternateTerm);

					suggestiveResult.setEpr(search(alternateTerm));

				} else {

					logger.debug("getting epr for " + query);

					suggestiveResult.setEpr(search(query));

				}

			} else {

				logger.debug("getting epr for " + query);

				suggestiveResult.setEpr(search(query));

			}

 		} catch (IndexServiceException ise) {

			logger.error("Error calling index", ise);

			throw new SearchServiceException("Error calling index.");

		}

		return suggestiveResult; */

    }

    private String getCorrectLocale(String givenLocaleName){

        String correctLocale = null;

        if (givenLocaleName == null) {

            return transformerFactory.getConfig().getDefaultLocale().getLanguage() + "_" + transformerFactory.getConfig().getDefaultLocale().getCountry();

        } else {

            correctLocale = givenLocaleName;

        }

        //get the locale

        StringTokenizer tokenizer = new StringTokenizer(correctLocale, "_");

        Locale requestLocale = new Locale(tokenizer.nextToken(), tokenizer.nextToken());

        if(requestLocale == null || !transformerFactory.getConfig().getLocales().contains(requestLocale)){

            correctLocale = transformerFactory.getConfig().getDefaultLocale().getLanguage() + "_" + transformerFactory.getConfig().getDefaultLocale().getCountry();

        }

        return correctLocale;

    }

    @Override

    public SearchResult search(String text, String transformer, String locale, int page, int size) throws SearchServiceException {

        logger.debug("Received Query: " + text);

        tlogger.debug("Received query " + text);

        IndexService index = getIndexLocator().getService();

        EPR epr = null;

        String query = rewrite(text);

        List<String> searchResults = null;

        try {

            long time = System.currentTimeMillis();

            cqlParser = new CQLParser();

            query = cqlParser.parse(query).toCQL();

            if (logger.isDebugEnabled()) {

                logger.debug("index lookup (all, query=" + query + ", " + mdFormat + ", index)");

            }

            /**

             * Ask index for search results

             **/

            epr = index.indexLookup("all", query, mdFormat, indexLayout);

            if (logger.isDebugEnabled()) {

                logger.debug("epr = " + epr);

            }

            time = System.currentTimeMillis() - time;

            logger.debug("index query lasted " + time + " msec");

            tlogger.debug("Got index response for query " + query);

        } catch (IndexServiceException ise) {

            logger.error("Error calling index.", ise);

            throw new SearchServiceException("Error calling index.");

        } catch (CQLParseException cqle) {

            logger.error("Bad CQL query.", cqle);

            throw new SearchServiceException("Error calling index.");

        } catch (IOException ioe) {

            logger.error("Bad CQL query.", ioe);

            throw new SearchServiceException("Error calling index.");

        }

        if (epr == null) {

            throw new SearchServiceException("Index returned null result set id.");

        }

        String correctLocale = getCorrectLocale(locale);

        StringTokenizer tokenizer = new StringTokenizer(correctLocale, "_");

        Locale requestLocale = new Locale(tokenizer.nextToken(), tokenizer.nextToken());

        /**

         * Read ResultSet for search

         */

        ResultSet<String> rs = rsFactory.createResultSet(epr);

        try {

            int rsSize = rs.size();

            int from = (page-1)*size + 1;

            int to = (page*size < rsSize) ? page*size : rsSize;

            List<String> xmls = rs.getElements(from, to);

            searchResults = new ArrayList<String>();

            Transformer tr = transformerFactory.getTransformer(transformer, requestLocale);

            /**

             * Transform the search records xmls

             */

            if (tr != null) {

                for (String xml:xmls) {

                    //logger.debug("xml index " + xml);

                    searchResults.add(tr.transform(xml));

                }

            } else {  // if there is no transformer defined add the xmls as you got them

                for (String xml:xmls) {

                    searchResults.add(xml);

                }

            }

        } catch (TransformerException te) {

            logger.error("Error transforming search results.", te);

            throw new SearchServiceException("Error transforming search results.", te);

        }

        logger.debug("Search results for query "+ query + " created.");

        logger.debug("Returned results for query " + query);

        return new SearchResult(query, correctLocale, rs.size(), page, size, searchResults);

    }

    @Override

    public SearchResult refine(String text, String transformer, String locale, Collection<String> fields) throws SearchServiceException {

        IndexService index = getIndexLocator().getService();

        EPR epr = null;

        String query = rewrite(text);

        List<String> browseResults = null;

        ResultSet<String> rs = null;

        /**

         * For refine

         */

        try {

            long time = System.currentTimeMillis();

            logger.debug("Refine query is: '"+query+"'");

            cqlParser = new CQLParser();

            query = cqlParser.parse(query).toCQL();

            logger.debug("The refine query " + query);

            StringBuffer buffer = new StringBuffer();

            buffer.append("query=").append(query).append("&groupby=");

            for (Iterator<String> iter = fields.iterator(); iter.hasNext();) {

                String field = (String) iter.next();

                buffer.append(field);

                if (iter.hasNext()) {

                    buffer.append(",");

                }

            }

            if (logger.isDebugEnabled()) {

                logger.debug("The refine query " + query);

                logger.debug("index refine (" + buffer.toString()

                        + ", all, " + mdFormat + ", index)");

            }

            epr = index.getBrowsingStatistics(buffer.toString(), "all", mdFormat, indexLayout);

            time = System.currentTimeMillis() - time;

            logger.debug("index query lasted " + time + " msec");

        } catch (CQLParseException cqle) {

            logger.error("Bad CQL query.", cqle);

            throw new SearchServiceException("Error calling index.");

        } catch (IndexServiceException ise) {

            logger.error("Error getting refine results.", ise);

            throw new SearchServiceException("Error getting refine results.", ise);

        } catch (IOException ioe) {

            logger.error("Bad CQL query.", ioe);

            throw new SearchServiceException("Error calling index.");

        }

        if (epr == null) {

            throw new SearchServiceException("Index returned null result set id.");

        }

        //get the locale

        String correctLocale = getCorrectLocale(locale);

        StringTokenizer tokenizer = new StringTokenizer(correctLocale, "_");

        Locale requestLocale = new Locale(tokenizer.nextToken(), tokenizer.nextToken());

        try{

            rs = rsFactory.createResultSet(epr);

            if (logger.isDebugEnabled()) {

                logger.debug("EPR : " + epr.getEpr());

            }

            List<String> list = rs.getElements(1, rs.size());

            logger.debug("record list size " + list.size());

            /**

             * Transform each refine row

             */

            Transformer tr = transformerFactory.getTransformer(transformer, requestLocale);

            browseResults = new ArrayList<String>();

            for (String row: list) {

                browseResults.add(tr.transform(row));

            }

        } catch (TransformerException te) {

            logger.error("Error transforming refine results.", te);

            throw new SearchServiceException("Error transforming refine results.", te);

        }

        return new SearchResult(query, correctLocale, fields, browseResults);

    }

    @Override

    public SearchResult searchNrefine(String text, String searchTransformer, String browseTransformer,

                                      String locale, int page, int size, Collection<String> fields) throws SearchServiceException {

        logger.info("Received Query: " + text);

        IndexService index = getIndexLocator().getService();

        EPR epr = null;

        String query = rewrite(text);

        //get the locale

        String correctLocale = getCorrectLocale(locale);

        StringTokenizer tokenizer = new StringTokenizer(correctLocale, "_");

        Locale requestLocale = new Locale(tokenizer.nextToken(), tokenizer.nextToken());

        /**

         * search

         * **/

        List<String> searchResults = null;

        try {

            long time = System.currentTimeMillis();

            cqlParser = new CQLParser();

            query = cqlParser.parse(query).toCQL();

            logger.debug("cqlParser returned query: " + query);

            if (logger.isDebugEnabled()) {

                logger.debug("index lookup (all, query=" + query + ", " + mdFormat + ", index)");

            }

            /**

             * Ask index for search results

             **/

            epr = index.indexLookup("all", query, mdFormat, "index");

            if (logger.isDebugEnabled()) {

                logger.debug("epr = " + epr);

            }

            time = System.currentTimeMillis() - time;

            logger.debug("index query lasted " + time + " msec");

        } catch (IndexServiceException ise) {

            logger.error("Error calling index.", ise);

            throw new SearchServiceException("Error calling index.");

        } catch (CQLParseException cqle) {

            logger.error("Bad CQL query.", cqle);

            throw new SearchServiceException("Error calling index.");

        } catch (IOException ioe) {

            logger.error("Bad CQL query.", ioe);

            throw new SearchServiceException("Error calling index.");

        }

        if (epr == null) {

            throw new SearchServiceException("Index returned null result set id.");

        }

        int rsSize = -1;

        /**

         * Read ResultSet for search

         */

        ResultSet<String> rs = rsFactory.createResultSet(epr);

        try {

            rsSize = rs.size();

            int from = (page-1)*size + 1;

            int to = (page*size < rsSize) ? page*size : rsSize;

            List<String> xmls = rs.getElements(from, to);

            Transformer tr = transformerFactory.getTransformer(searchTransformer, requestLocale);

            searchResults = new ArrayList<String>();

            /**

             * Transform the search records xmls

             */

            for (String xml:xmls) {

                searchResults.add(tr.transform(xml));

            }

        } catch (TransformerException te) {

            logger.error("Error transforming search results.", te);

            throw new SearchServiceException("Error transforming search results.", te);

        }

        /**

         * refine

         */

        List<String> browseResults = null;

        try {

            long time = System.currentTimeMillis();

            cqlParser = new CQLParser();

            query = cqlParser.parse(query).toCQL();

            StringBuffer buffer = new StringBuffer();

            buffer.append("query=").append(query).append("&groupby=");

            for (Iterator<String> iter = fields.iterator(); iter.hasNext();) {

                String field = (String) iter.next();

                buffer.append(field);

                if (iter.hasNext()) {

                    buffer.append(",");

                }

            }

            if (logger.isDebugEnabled()) {

                logger.debug("index refine (" + buffer.toString()

                        + ", all, " + mdFormat + ", index)");

            }

            epr = index.getBrowsingStatistics(buffer.toString(), "all", mdFormat, "index");

            time = System.currentTimeMillis() - time;

            logger.debug("index query lasted " + time + " msec");

        } catch (CQLParseException cqle) {

            logger.error("Bad CQL query.", cqle);

            throw new SearchServiceException("Error calling index.");

        } catch (IOException ioe) {

            logger.error("Bad CQL query.", ioe);

            throw new SearchServiceException("Error calling index.");

        } catch (IndexServiceException ise) {

            logger.error("Error getting refine results.", ise);

            throw new SearchServiceException("Error getting refine results.", ise);

        }

        if (epr == null) {

            throw new SearchServiceException("Index returned null result set id.");

        }

        try{

            rs = rsFactory.createResultSet(epr);

            if (logger.isDebugEnabled()) {

                logger.debug("EPR : " + epr.getEpr());

            }

            List<String> list = rs.getElements(1, rs.size());

            logger.debug("record list size " + list.size());

            Transformer tr = transformerFactory.getTransformer(browseTransformer, requestLocale);

            browseResults = new ArrayList<String>();

            for (String row: list) {

                browseResults.add(tr.transform(row));

            }

        } catch (TransformerException te) {

            logger.error("Error transforming refine results.", te);

            throw new SearchServiceException("Error transforming refine results.", te);

        }

        return new SearchResult(query,  correctLocale, rsSize, page, size, searchResults, browseResults, fields);

    }

    public List<String> getSearchResultsFromIndex(){

        return null;

    }

    public List<String> getRefineResultsFromIndex(){

        return null;

    }

    @Override

    public FormattedSearchResult search(String queryText, String transformer,

                                        String format, String locale, int page, int size)

            throws SearchServiceException {

        FormattedSearchResult formattedSearchResult = null;

        SearchResult searchResult = search(queryText, transformer, locale, page, size);

        Formatter formatter = transformerFactory.getFormatter(format);

        if (formatter == null) {

            logger.error("Error formatting search results. " + format +" formatter does not exist.");

            throw new SearchServiceException("Error formatting search results. " + format +" formatter does not exist.");

        }

        try {

            formattedSearchResult = new FormattedSearchResult(formatter.format(searchResult), searchResult.getTotal());

            logger.debug("Returning formatted result: page " + page + ", size: " + size + " of total: " + searchResult.getTotal());

        } catch (Exception e) {

            logger.error("Error formatting search results.", e);

        }

        return formattedSearchResult;

    }

    @Override

    public FormattedSearchResult refine(String queryText, String refineTransformer,

                                        String format, String locale, Collection<String> fields) throws SearchServiceException {

        FormattedSearchResult formattedSearchResult = null;

        SearchResult searchResult = refine(queryText, refineTransformer, locale, fields);

        Formatter formatter = transformerFactory.getFormatter(format);

        if (formatter == null) {

            logger.error("Error formatting refine results. " + format +" formatter does not exist.");

            throw new SearchServiceException("Error formatting refine results. " + format +" formatter does not exist.");

        }

        try {

            formattedSearchResult = new FormattedSearchResult(formatter.format(searchResult), searchResult.getTotal());

        } catch (Exception e) {

            logger.error("Error formatting refine results.", e);

        }

        return formattedSearchResult;

    }

    @Override

    public FormattedSearchResult searchNrefine(String queryText,

                                               String searchTransformer, String refineTransformer, String format,

                                               String locale, int page, int size, Collection<String> fields)

            throws SearchServiceException {

        FormattedSearchResult formattedSearchResult = null;

        SearchResult searchResult = searchNrefine(queryText, searchTransformer, refineTransformer, locale, page, size, fields);

        Formatter formatter = transformerFactory.getFormatter(format);

        if (formatter == null) {

            logger.error("Error formating search and refine results. " + format +" formatter does not exist.");

            throw new SearchServiceException("Error formating search and refine results. " + format +" formatter does not exist.");

        }

        try {

            formattedSearchResult = new FormattedSearchResult(formatter.format(searchResult), searchResult.getTotal());

        } catch (Exception e) {

            logger.error("Error formating search results.", e);

        }

        return formattedSearchResult;

    }

    private String rewrite(String query) {

        if (queryRules != null) {

            for (QueryRewriteRule queryRule: queryRules) {

                if (logger.isDebugEnabled()) {

                    logger.debug("Apply rule " + query);

                }

                query = queryRule.apply(query);

                if (logger.isDebugEnabled()) {

                    logger.debug("Rewritten query is " + query);

                }

            }

        }

        return query;

    }

    public String getMdFormat() {

        return mdFormat;

    }

    public void setMdFormat(String mdFormat) {

        this.mdFormat = mdFormat;

    }

    public ServiceLocator<IndexService> getIndexLocator() {

        return indexLocator;

    }

    public void setIndexLocator(ServiceLocator<IndexService> indexLocator) {

        this.indexLocator = indexLocator;

    }

    public ResultSetFactory getRsFactory() {

        return rsFactory;

    }

    public void setRsFactory(ResultSetFactory rsFactory) {

        this.rsFactory = rsFactory;

    }

    public Collection<FieldRewriteRule> getFieldRules() {

        return fieldRules.values();

    }

    public void setFieldRules(Collection<FieldRewriteRule> fieldRules) {

        this.fieldRules = new HashMap<String, FieldRewriteRule>();

        for (FieldRewriteRule rule : fieldRules) {

            String key = rule.getFieldName();

            if (this.fieldRules.containsKey(key)) {

                logger.warn("Multiple rules for field " + key);

                logger.warn("Keeping last rule " + rule.getName());

            }

            this.fieldRules.put(key, rule);

        }

    }

    public List<QueryRewriteRule> getQueryRules() {

        return queryRules;

    }

    public void setQueryRules(List<QueryRewriteRule> queryRules) {

        this.queryRules = queryRules;

    }

    public boolean isEnableBrowseCache() {

        return enableBrowseCache;

    }

    public void setEnableBrowseCache(boolean enableBrowseCache) {

        this.enableBrowseCache = enableBrowseCache;

    }

    public SearchRegistry getTransformerFactory() {

        return transformerFactory;

    }

    public void setTransformerFactory(SearchRegistry transformerFactory) {

        this.transformerFactory = transformerFactory;

    }

    public String getIndexLayout() {

        return indexLayout;

    }

    public void setIndexLayout(String indexLayout) {

        this.indexLayout = indexLayout;

    }

    public SearchServiceBlackboardHandler getBlackboardNotificationHandler() {

        return blackboardNotificationHandler;

    }

    public void setBlackboardNotificationHandler(SearchServiceBlackboardHandler blackboardNotificationHandler) {

        this.blackboardNotificationHandler = blackboardNotificationHandler;

    }

    public void setLookUpServiceServiceLocator(ServiceLocator<ISLookUpService> lookUpServiceServiceLocator) {

        this.lookUpServiceServiceLocator = lookUpServiceServiceLocator;

    }

}