D-Net

package eu.dnetlib.data.search.utils.vocabulary;

public class VocabularyFactoryImpl {

	public static void createXmlVocabulary(Vocabulary vocabulary) {

	}

}

package eu.dnetlib.data.search.utils.vocabulary;

import gr.uoa.di.driver.xml.VocabularyXmlConverter;

import org.apache.commons.io.IOUtils;

import org.apache.log4j.Logger;

import javax.xml.bind.JAXBException;

import java.io.IOException;

import java.io.InputStream;

import java.util.Locale;

import java.util.StringTokenizer;

/**

 * Loads vocabulary from the classpath

 * @author kiatrop

 *

 */

public class LocalVocabularyLoader extends VocabularyLoader {

	private static final Logger logger = Logger.getLogger(LocalVocabularyLoader.class);

	@Override

	public eu.dnetlib.domain.enabling.Vocabulary getVocabulary(Vocabulary vocabulary, Locale locale) {

		logger.debug("Getting vocabulary with name " + vocabulary.getName() + " and locale " + locale);

		String xml = null;

		StringTokenizer tokenizer = new StringTokenizer(((LocalVocabulary)vocabulary).getFileName(), ".");

		String localizedVocabularyName = tokenizer.nextToken() +  "_" +

				locale.getLanguage() + "_" + locale.getCountry() + "." + tokenizer.nextToken();

		InputStream xmlInputStream = this.getClass().getClassLoader().getResourceAsStream(localizedVocabularyName);

		if (xmlInputStream == null) {

			logger.debug("The " + localizedVocabularyName + " does not exist in classpath. Loading default vocabulary instead.");

			xmlInputStream = this.getClass().getClassLoader().getResourceAsStream(((LocalVocabulary)vocabulary).getFileName());

		}

		if (xmlInputStream == null) {

			logger.warn("The " + ((LocalVocabulary)vocabulary).getFileName() + " does not exist in classpath.");

			return null; //null

		}

		try {

			xml = IOUtils.toString(xmlInputStream, "UTF-8");

			if (xml!=null && !xml.trim().isEmpty()) {

				VocabularyXmlConverter converter = null;

				converter = new VocabularyXmlConverter();

				return converter.XmlToObject(xml);

			}

		} catch (IOException ioe) {

			logger.debug("Problem loading " + localizedVocabularyName + " from classpath", ioe);

		} catch (JAXBException jaxbe) {

			logger.error("Unable to load vocabulary " + vocabulary.getName(), jaxbe);

		} finally {

            IOUtils.closeQuietly(xmlInputStream);

        }

		return null;

	}

}

package eu.dnetlib.data.search.utils.vocabulary;

public class LocalVocabulary extends Vocabulary{

	/** the name of the vocabulary in classpath */

	private final String fileName;

	public LocalVocabulary(String name, String filename) {

		super(name);

		this.fileName = filename;

	}

	public String getFileName() {

		return fileName;

	}

}

package eu.dnetlib.data.search.utils.vocabulary;

public abstract class Vocabulary {

	private final String name;

	public String getName() {

		return name;

	}

	public Vocabulary(String name) {

		this.name = name;

	}

}

package eu.dnetlib.data.search.micrometer;

import org.springframework.stereotype.Component;

import org.springframework.web.servlet.handler.HandlerInterceptorAdapter;

@Component

public class PrometheusRequestTimerInterceptor extends HandlerInterceptorAdapter{

    /*private PrometheusMeterRegistry prometheusRegistry;

    private DistributionSummary summary;

    PrometheusRequestTimerInterceptor(){}

    PrometheusRequestTimerInterceptor(PrometheusMeterRegistry prometheusMeterRegistry) {

        this.prometheusRegistry = prometheusMeterRegistry;

        summary = DistributionSummary.builder("search.server.response.size")

                .publishPercentileHistogram()

                .sla(1000000, 2500000, 3500000)

                .minimumExpectedValue((long) 1000000)

                .maximumExpectedValue((long) 3500000)

                .scale(500000)

                .register(prometheusRegistry);

    }

    @Override

    public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex)

            throws Exception {

        summary.record((double)response.getBufferSize());

    }

    @Override

    public void postHandle(

            HttpServletRequest request,

            HttpServletResponse response,

            Object handler,

            ModelAndView modelAndView) throws Exception {

    }*/

}

<?xml version="1.0"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>uoa-search</artifactId>

	<packaging>jar</packaging>

	<version>3.8.3</version>

	<scm>

                <developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/uoa-search/tags/uoa-search-3.8.3</developerConnection> 

        </scm>

    <repositories>

        <!-- Cloudera Repositories -->

        <repository>

            <snapshots>

                <enabled>false</enabled>

            </snapshots>

            <id>cloudera-central</id>

            <name>cloundera-libs-release</name>

            <url>http://maven.research-infrastructures.eu/nexus/content/repositories/cloudera-central</url>

        </repository>

        <repository>

            <id>cloudera-snapshots</id>

            <name>cloudera-libs-snapshot</name>

            <url>http://maven.research-infrastructures.eu/nexus/content/repositories/cloudera-snapshots</url>

        </repository>

        <repository>

            <id>typesafe</id>

            <name>typesafe-releases</name>

            <url>http://maven.research-infrastructures.eu/nexus/content/repositories/typesafe</url>

        </repository>

    </repositories>

	<dependencies>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>uoa-commons</artifactId>

			<version>[2.0.0, 3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>org.mockito</groupId>

			<artifactId>mockito-all</artifactId>

			<version>1.6</version>

		</dependency>

		<dependency>

			<groupId>org.apache.solr</groupId>

			<artifactId>solr-solrj</artifactId>

			<version>7.2.1</version>

            <exclusions>

                <exclusion>

                    <groupId>org.apache.zookeeper</groupId>

                    <artifactId>zookeeper</artifactId>

                </exclusion>

            </exclusions>

		</dependency>

        <dependency>

            <groupId>org.apache.zookeeper</groupId>

            <artifactId>zookeeper</artifactId>

            <version>3.4.5-cdh4.3.0</version>

        </dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-cql-utils</artifactId>

			<version>[2.0.0, 3.0.0)</version>

			<scope>compile</scope>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-blackboard-common</artifactId>

			<version>[2.2.1, 3.0.0)</version>

		</dependency>

		<!-- Spring MVC framework -->

		<dependency>

			<groupId>org.springframework</groupId>

			<artifactId>spring-webmvc</artifactId>

			<version>${spring.version}</version>

		</dependency>

		<dependency>

			<groupId>org.apache.velocity</groupId>

			<artifactId>velocity</artifactId>

			<version>1.7</version>

		</dependency>

		<dependency>

			<groupId>asm</groupId>

			<artifactId>asm</artifactId>

			<version>3.3.1</version>

		</dependency>

		<dependency>

			<groupId>com.sun.jersey</groupId>

			<artifactId>jersey-bundle</artifactId>

			<version>1.19</version>

		</dependency>

		<dependency>

			<groupId>org.json</groupId>

			<artifactId>json</artifactId>

			<version>20140107</version>

		</dependency>

		<dependency>

			<groupId>com.sun.jersey</groupId>

			<artifactId>jersey-server</artifactId>

			<version>1.19</version>

		</dependency>

		<dependency>

			<groupId>com.sun.jersey</groupId>

			<artifactId>jersey-core</artifactId>

			<version>1.19</version>

		</dependency>

		<dependency>

			<groupId>commons-validator</groupId>

			<artifactId>commons-validator</artifactId>

			<version>1.6</version>

		</dependency>

		<dependency>

			<groupId>org.eclipse.persistence</groupId>

			<artifactId>org.eclipse.persistence.moxy</artifactId>

			<version>2.5.0</version>

		</dependency>

		<dependency>

			<groupId>com.sun.jersey.contribs</groupId>

			<artifactId>jersey-spring</artifactId>

			<version>1.8</version>

			<exclusions>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring-core</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring-web</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring-beans</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring-context</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring-utils</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring-aop</artifactId>

				</exclusion>

			</exclusions>

		</dependency>

		<!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->

		<dependency>

			<groupId>com.google.code.gson</groupId>

			<artifactId>gson</artifactId>

			<version>2.3.1</version>

		</dependency>

		<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->

		<dependency>

			<groupId>org.apache.commons</groupId>

			<artifactId>commons-lang3</artifactId>

			<version>3.5</version>

		</dependency>

		<dependency>

			<groupId>io.prometheus</groupId>

			<artifactId>simpleclient</artifactId>

			<version>0.6.0</version>

		</dependency>

		<dependency>

			<groupId>io.prometheus</groupId>

			<artifactId>simpleclient_servlet</artifactId>

			<version>0.6.0</version>

		</dependency>

		<dependency>

			<groupId>io.prometheus</groupId>

			<artifactId>simpleclient_hotspot</artifactId>

			<version>0.6.0</version>

		</dependency>

		<!-- https://mvnrepository.com/artifact/io.micrometer/micrometer-registry-prometheus -->

		<dependency>

			<groupId>io.micrometer</groupId>

			<artifactId>micrometer-registry-prometheus</artifactId>

			<version>1.2.0</version>

		</dependency>

		<!-- https://mvnrepository.com/artifact/io.micrometer/micrometer-spring-legacy -->

		<dependency>

			<groupId>io.micrometer</groupId>

			<artifactId>micrometer-spring-legacy</artifactId>

			<version>1.2.0</version>

		</dependency>

		<!-- https://mvnrepository.com/artifact/org.aspectj/aspectjrt -->

		<!-- AspectJ -->

		<dependency>

			<groupId>org.aspectj</groupId>

			<artifactId>aspectjrt</artifactId>

			<version>1.9.4</version>

		</dependency>

		<dependency>

			<groupId>org.aspectj</groupId>

			<artifactId>aspectjweaver</artifactId>

			<version>1.9.4</version>

		</dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.maven/maven-model -->

        <dependency>

            <groupId>org.apache.maven</groupId>

            <artifactId>maven-model</artifactId>

            <version>3.3.9</version>

        </dependency>

    </dependencies>

	<build>

		<plugins>

			<plugin>

				<groupId>org.apache.maven.plugins</groupId>

				<artifactId>maven-compiler-plugin</artifactId>

				<configuration>

					<source>1.8</source>

					<target>1.8</target>

				</configuration>

			</plugin>

		</plugins>

	</build>

</project>

package eu.dnetlib.data.search.app;

import eu.dnetlib.api.data.IndexService;

import eu.dnetlib.api.data.IndexServiceException;

import eu.dnetlib.api.data.SearchService;

import eu.dnetlib.api.data.SearchServiceException;

import eu.dnetlib.api.enabling.ISLookUpService;

import eu.dnetlib.common.rmi.UnimplementedException;

import eu.dnetlib.data.search.app.plan.FieldRewriteRule;

import eu.dnetlib.data.search.app.plan.QueryRewriteRule;

import eu.dnetlib.data.search.solr.SolrResultSet;

import eu.dnetlib.data.search.transform.Transformer;

import eu.dnetlib.data.search.transform.config.SearchRegistry;

import eu.dnetlib.data.search.transform.formatter.Formatter;

import eu.dnetlib.domain.ActionType;

import eu.dnetlib.domain.EPR;

import eu.dnetlib.domain.ResourceType;

import eu.dnetlib.domain.data.FormattedSearchResult;

import eu.dnetlib.domain.data.SearchResult;

import eu.dnetlib.domain.data.SuggestiveResult;

import eu.dnetlib.domain.enabling.Notification;

import gr.uoa.di.driver.app.DriverServiceImpl;

import gr.uoa.di.driver.enabling.issn.NotificationListener;

import gr.uoa.di.driver.enabling.resultset.ResultSet;

import gr.uoa.di.driver.enabling.resultset.ResultSetFactory;

import gr.uoa.di.driver.util.ServiceLocator;

import io.micrometer.core.instrument.Timer;

import io.micrometer.prometheus.PrometheusMeterRegistry;

import org.apache.log4j.Logger;

import org.apache.solr.client.solrj.SolrServerException;

import org.springframework.beans.factory.annotation.Autowired;

import org.w3c.dom.Document;

import org.w3c.dom.Node;

import org.xml.sax.InputSource;

import javax.xml.parsers.DocumentBuilder;

import javax.xml.parsers.DocumentBuilderFactory;

import javax.xml.xpath.XPath;

import javax.xml.xpath.XPathConstants;

import javax.xml.xpath.XPathExpression;

import javax.xml.xpath.XPathFactory;

import java.io.OutputStream;

import java.io.StringReader;

import java.util.*;

//import eu.dnetlib.utils.cql.CqlException;

public class SearchServiceImpl extends DriverServiceImpl

        implements SearchService {

    private static Logger logger = Logger.getLogger(SearchServiceImpl.class);

    //@Deprecated

    //private static Logger tlogger = Logger.getLogger("eu.dnetlib.data.search.app.Timer");

    private String mdFormat = "DMF";

    private String indexLayout = "index";

    private ServiceLocator<IndexService> indexLocator = null;

    private ServiceLocator<ISLookUpService> lookUpServiceServiceLocator = null;

    private ResultSetFactory rsFactory = null;

    private SearchRegistry transformerFactory = null;

    private List<QueryRewriteRule> queryRules = null;

    private List<String> fieldQueryRules = null;

    private Map<String, FieldRewriteRule> fieldRules = null;

    private boolean enableBrowseCache = false;

    private SearchServiceBlackboardHandler blackboardNotificationHandler = null;

    @Autowired

    private PrometheusMeterRegistry registry;

    //private CQLParser cqlParser = null;

    @Override

    public void init() {

        super.init();

        String serviceId = this.getServiceEPR().getParameter("serviceId");

        this.subscribe(

                ActionType.UPDATE,

                ResourceType.ANY.SEARCHSERVICERESOURCETYPE,

                serviceId,

                "RESOURCE_PROFILE/BODY/BLACKBOARD/LAST_REQUEST",

                new NotificationListener() {

                    @Override

                    public void processNotification(Notification notification) {

                        blackboardNotificationHandler.notified(

                                notification.getSubscriptionId(),

                                notification.getTopic(),

                                notification.getIsId(),

                                notification.getMessage());

                    }

                });

        try {

            String searchProfile = lookUpServiceServiceLocator.getService().getResourceProfile(serviceId);

            if (searchProfile != null) {

                DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();

                dbf.setNamespaceAware(true);

                DocumentBuilder db = dbf.newDocumentBuilder();

                Document doc = db.parse(new InputSource(new StringReader(searchProfile)));

                XPathFactory factory = XPathFactory.newInstance();

                XPath xpath = factory.newXPath();

                XPathExpression searchMdFormatExpression = xpath.compile("//SERVICE_PROPERTIES/PROPERTY[@key='mdformat']");

                Node node = (Node) searchMdFormatExpression.evaluate(doc,XPathConstants.NODE);

                if (node != null){

                    String profileMdFormat = node.getAttributes().getNamedItem("value").getTextContent();

                    if (profileMdFormat != null) {

                        //logger.debug("mdformat in properties " + mdFormat );

                        logger.info("Setting mdformat to '" + profileMdFormat + "'");

                        mdFormat = profileMdFormat;

                    }

                }

            }

        } catch (Exception e) {

            logger.error("Fail to load search service profile with id " + serviceId + " from IS.", e);

        }

    }

    @Override

    public SuggestiveResult suggestiveSearch(String query) throws SearchServiceException {

        throw new UnimplementedException();

    }

    @Override

    @Deprecated

    public SearchResult search(String text, String transformer, String locale, int page, int size) throws SearchServiceException {

        return searchNrefine(text, transformer, null, locale, page, size, null);

    }

    @Override

    @Deprecated

    public SearchResult refine(String text, String transformer, String locale, Collection<String> fields) throws SearchServiceException {

        return searchNrefine(text, null, transformer, locale, 1, -1, fields);

    }

    @Override

    @Deprecated

    public SearchResult searchNrefine(String text, String searchTransformer, String browseTransformer,

                                      String locale, int page, int size, Collection<String> fields) throws SearchServiceException {

        //logger.info("deprecated searchNrefine > from: " + page + " to:" + size);

        //TODO check locale

        //logger.debug("Search transformer " + searchTransformer);

        Transformer sTransformer = transformerFactory.getTransformer(searchTransformer, Locale.getDefault());

        Transformer oldRefineTransformer = transformerFactory.getTransformer("results_openaire_browse", Locale.getDefault());

        //logger.debug("Refine transformer " + browseTransformer);

        //Transformer rTranformer = transformerFactory.getTransformer(refineTransformer, Locale.getDefault());

        List<String> refineFields = null;

        if (fields!=null) {

            refineFields = new ArrayList<String>(fields);

        }

        return newSearch(text, locale, refineFields, null, new ArrayList<String>(), page, size, "", sTransformer, oldRefineTransformer, true);

    }

    @Override

    public FormattedSearchResult search(String queryText, String transformerName, String format, String locale, int page, int size)

            throws SearchServiceException {

        return searchNrefine(queryText, transformerName, null, format, locale, page, size, null);

    }

    @Override

    public FormattedSearchResult refine(String queryText, String refineTransformer, String format, String locale, Collection<String> fields) throws SearchServiceException {

        return searchNrefine(queryText, null, refineTransformer, format, locale, 0, -1, fields);

    }

    @Override

    public FormattedSearchResult searchNrefine(String queryText,

                                               String searchTransformer, String refineTransformer, String format,

                                               String locale, int page, int size, Collection<String> fields)  throws SearchServiceException {

        //logger.info("searchNrefine > from: " + page + " to:" + size);

        //TODO check locale

        FormattedSearchResult formattedSearchResult = null;

        //logger.debug("Search transformer " + searchTransformer);

        Transformer sTransformer = transformerFactory.getTransformer(searchTransformer, Locale.getDefault());

        Transformer oldRefineTransformer = transformerFactory.getTransformer("results_openaire_browse", Locale.getDefault());

        //logger.debug("Refine transformer " + refineTransformer);

        //Transformer rTranformer = transformerFactory.getTransformer(refineTransformer, Locale.getDefault());

        List<String> refineFields = null;

        if (fields!=null) {

            refineFields = new ArrayList<String>(fields);

        }

        SearchResult searchResult = newSearch(queryText, locale, refineFields, new ArrayList<>(), new ArrayList<String>(), page, size, format, sTransformer, oldRefineTransformer, true);

        Formatter formatter = transformerFactory.getFormatter(format); // formatter cannot be returned as null

        try {

            formattedSearchResult = new FormattedSearchResult(formatter.format(searchResult), searchResult.getTotal());

        } catch (Exception e) {

            logger.error("Error formating search results.", e);

        }

        return formattedSearchResult;

    }

    public SearchResult newSearch (String text, String locale, List<String> refinefields, List<String> specialFacets, List<String> fieldQueries,

                                   int from, int to, String format, Transformer transformer, Transformer oldRefineTransformer,

                                   boolean oldPaging) throws SearchServiceException {

        logger.info("newSearch > from: " + from + " to:" + to);

        long startTime = System.nanoTime();

        Timer.Sample timer = Timer.start(registry);

        IndexService index = getIndexLocator().getService();

        EPR epr = null;

        ResultSet<String> rs = null;

        List<String> browseResults = null;

        List<String> searchResults = null;

        String query = rewrite(text);

        enhanceFieldQueries(fieldQueries);

        logger.info("Performing query '" + query + "' and fields " + fieldQueries + " and refine " + refinefields);

        try {

            //TODO see parser and maybe delete!

            //query = new CQLParser().parse(query).toCQL();

            String eprQuery = createEprQuery(query, refinefields, specialFacets, fieldQueries);

            epr = index.getBrowsingStatistics(eprQuery, "all", mdFormat, indexLayout);

            if (epr == null){

                throw new SearchServiceException("Something really strange happened there! Index returned null result set id.");

            }

            //get the locale TODO do we need this?

            //String correctLocale = getCorrectLocale(locale);

            //StringTokenizer tokenizer = new StringTokenizer(correctLocale, "_");

            //Locale requestLocale = new Locale(tokenizer.nextToken(), tokenizer.nextToken());

            rs = rsFactory.createResultSet(epr);

            Map<String, List<String>> list = null;

            if (oldPaging) {

                list = ((SolrResultSet)rs).newGet(from-1, to, format, transformer, oldRefineTransformer);

            } else {

                list = ((SolrResultSet)rs).newGet(from, to, format, transformer, oldRefineTransformer);

            }

            searchResults = list.get("search");

            browseResults = list.get("refine");

        } catch (IndexServiceException ise) {

            logger.error("Error getting refine results.", ise);

            throw new SearchServiceException("Error getting refine results.", ise);

        } finally {

            timer.stop(registry.timer("search.server.response.duration"));

/*

            histogram.observe(System.currentTimeMillis()-startTime);

*/

        }

        long estimatedTime = System.nanoTime() - startTime;

        logger.debug("Search time " + estimatedTime/1000000 +  " milliseconds for query " + query +

                " and fields " + fieldQueries + " and refine " + refinefields + " from: "+ from + " and size " + to);

        //logger.info("Returned results for NEW search query '" + query + "' and fields " + fieldQueries + " and refine " + refinefields);;

        rs.close();

        return new SearchResult(query, Locale.getDefault().toString(), rs.size(), from, to, searchResults, browseResults, refinefields);

    }

    public void cursorSearch(String text, List<String> refinefields, List<String> specialFacets, List<String> fieldQueries,

                             String format, Transformer transformer, OutputStream os) throws SearchServiceException {

        long startTime = System.nanoTime();

        IndexService index = getIndexLocator().getService();

        EPR epr = null;

        ResultSet<String> rs = null;

        String query = rewrite(text);

        enhanceFieldQueries(fieldQueries);

        logger.info("Performing cursor query " + query + "' and fields " + fieldQueries + " and refine " + refinefields);

        logger.debug("Performing cursor query " + query + "' and fields " + fieldQueries + " and refine " + refinefields);

        try {

            String eprQuery = createEprQuery(query, refinefields, specialFacets, fieldQueries);

            epr = index.getBrowsingStatistics(eprQuery, "all", mdFormat, indexLayout);

            if (epr == null) {

                throw new SearchServiceException("Something really strange happened there! Index returned null result set id.");

            }

            rs = rsFactory.createResultSet(epr);

            ((SolrResultSet)rs).cursorGet(transformer,2000, os);

        } catch (IndexServiceException ise) {

            logger.error("Error getting cursor results.", ise);

            throw new SearchServiceException("Error getting cursor results.", ise);

        } catch (SolrServerException sse) {

            logger.error("Error getting cursor results.", sse);

            throw new SearchServiceException("Error getting cursor results.", sse);

        }

        long estimatedTime = System.nanoTime() - startTime;

        logger.debug("Cursor search time " + estimatedTime/1000000 +  " milliseconds for query " + query +

                " and fields " + fieldQueries + " and refine " + refinefields);

        rs.close();

    }

    private String rewrite(String query) {

        if (queryRules != null) {

            for (QueryRewriteRule queryRule: queryRules) {

                if (logger.isDebugEnabled()) {

                    logger.debug("Apply rule " + query);

                }

                query = queryRule.apply(query);

                if (logger.isDebugEnabled()) {

                    logger.debug("Rewritten query is " + query);

                }

            }

        }

        return query;

    }

    private void enhanceFieldQueries(List<String> fieldQueries) {

        if (fieldQueries != null && fieldQueryRules != null && !fieldQueryRules.isEmpty()) {

            fieldQueries.addAll(fieldQueryRules);

        }

    }

    public void setFieldRules(Collection<FieldRewriteRule> fieldRules) {

        this.fieldRules = new HashMap<String, FieldRewriteRule>();

        for (FieldRewriteRule rule : fieldRules) {

            String key = rule.getFieldName();

            if (this.fieldRules.containsKey(key)) {

                logger.warn("Multiple rules for field " + key);

                logger.warn("Keeping last rule " + rule.getName());

            }

            this.fieldRules.put(key, rule);

        }

    }

    public static String createEprQuery(String query, List<String> refineFields, List<String> specialFacets, List<String> fieldQueries) {

        StringBuffer queryBuffer = new StringBuffer();

        queryBuffer.append("query=");

        StringBuffer facetsBuffer = new StringBuffer();

        facetsBuffer.append("&groupby=");

        StringBuffer fqBuffer = new StringBuffer();

        fqBuffer.append("&fq=");

        StringBuffer sfBuffer = new StringBuffer();

        sfBuffer.append("&sf=");

        if (query != null) { //TODO consider exception?

            queryBuffer.append(query);

        }

        if(refineFields != null) {

            for (Iterator<String> iterator = refineFields.iterator(); iterator.hasNext(); ) {

                facetsBuffer.append(iterator.next());

                if (iterator.hasNext()) {

                    facetsBuffer.append(",");

                }

            }

        }

        if(specialFacets != null) {

            for (Iterator<String> iterator = specialFacets.iterator(); iterator.hasNext(); ) {

                sfBuffer.append(iterator.next());

                if (iterator.hasNext()) {

                    sfBuffer.append(",");

                }

            }

        }

       // logger.debug("special buffer " + sfBuffer.toString());

        if(fieldQueries != null) {

            for (Iterator<String> iterator = fieldQueries.iterator(); iterator.hasNext(); ) {

                fqBuffer.append(iterator.next());

                if (iterator.hasNext()) {

                    fqBuffer.append(",");

                }

            }

        }

        return queryBuffer.append(facetsBuffer.toString()).append(sfBuffer.toString()).append(fqBuffer.toString()).toString();

    }

    //TODO: I wish to remove this. This was only made (quick and dirty - only the enhanceFieldQueries(fieldQueries) is missing

    //from newSearch() after a last time request for the portal to show all the publications and the deletedbyinference ones.

    //I did not want to pass a parameter since I do not know if we are going to keep it. This is for a tech meeting showcase.

    //If we want to keep this I need to redesign.

    public SearchResult newSearchWithoutFieldQueries (String text, String locale, List<String> refinefields, List<String> specialFacets,

                                                      List<String> fieldQueries, int from, int to, String format, Transformer transformer,

                                                      Transformer oldRefineTransformer, boolean oldPaging) throws SearchServiceException {

        logger.info("non filtered search for...  > from: " + from + " to:" + to);

        long startTime = System.nanoTime();

        IndexService index = getIndexLocator().getService();

        EPR epr = null;

        ResultSet<String> rs = null;

        List<String> browseResults = null;

        List<String> searchResults = null;

        String query = rewrite(text);

        logger.info("Performing query '" + query + "' and fields " + fieldQueries + " and refine " + refinefields);

        try {

            //TODO see parser and maybe delete!

            //query = new CQLParser().parse(query).toCQL();

            String eprQuery = createEprQuery(query, refinefields, specialFacets, fieldQueries);

            epr = index.getBrowsingStatistics(eprQuery, "all", mdFormat, indexLayout);

            if (epr == null) {

                throw new SearchServiceException("Something really strange happened there! Index returned null result set id.");

            }

            //get the locale TODO do we need this?

            //String correctLocale = getCorrectLocale(locale);

            //StringTokenizer tokenizer = new StringTokenizer(correctLocale, "_");

            //Locale requestLocale = new Locale(tokenizer.nextToken(), tokenizer.nextToken());

            rs = rsFactory.createResultSet(epr);

            Map<String, List<String>> list = null;

            if (oldPaging) {

                list = ((SolrResultSet)rs).newGet(from-1, to, format, transformer, oldRefineTransformer);

            } else {

                list = ((SolrResultSet)rs).newGet(from, to, format, transformer, oldRefineTransformer);

            }

            searchResults = list.get("search");

            browseResults = list.get("refine");

        } catch (IndexServiceException ise) {

            logger.error("Error getting refine results.", ise);

            throw new SearchServiceException("Error getting refine results.", ise);

        }

        long estimatedTime = System.nanoTime() - startTime;

        logger.debug("Search time " + estimatedTime/1000000 +  " milliseconds for query " + query +

                " and fields " + fieldQueries + " and refine " + refinefields + " from: "+ from + " and size " + to);

        //logger.info("Returned results for NEW search query '" + query + "' and fields " + fieldQueries + " and refine " + refinefields);;

        rs.close();

        return new SearchResult(query, Locale.getDefault().toString(), rs.size(), from, to, searchResults, browseResults, refinefields);

    }

    public String getMdFormat() {

        return mdFormat;

    }

    public void setMdFormat(String mdFormat) {

        this.mdFormat = mdFormat;

    }

    public ServiceLocator<IndexService> getIndexLocator() {

        return indexLocator;

    }

    public void setIndexLocator(ServiceLocator<IndexService> indexLocator) {

        this.indexLocator = indexLocator;

    }

    public ResultSetFactory getRsFactory() {

        return rsFactory;

    }

    public void setRsFactory(ResultSetFactory rsFactory) {

        this.rsFactory = rsFactory;

    }

    public Collection<FieldRewriteRule> getFieldRules() {

        return fieldRules.values();

    }

    public List<QueryRewriteRule> getQueryRules() {

        return queryRules;

    }

    public void setQueryRules(List<QueryRewriteRule> queryRules) {

        this.queryRules = queryRules;

    }

    public boolean isEnableBrowseCache() {

        return enableBrowseCache;

    }

    public void setEnableBrowseCache(boolean enableBrowseCache) {

        this.enableBrowseCache = enableBrowseCache;

    }

    public SearchRegistry getTransformerFactory() {

        return transformerFactory;

    }

    public void setTransformerFactory(SearchRegistry transformerFactory) { this.transformerFactory = transformerFactory; }

    public String getIndexLayout() {

        return indexLayout;

    }

    public void setIndexLayout(String indexLayout) {

        this.indexLayout = indexLayout;

    }

    public SearchServiceBlackboardHandler getBlackboardNotificationHandler() {

        return blackboardNotificationHandler;

    }

    public void setBlackboardNotificationHandler(SearchServiceBlackboardHandler blackboardNotificationHandler) {

        this.blackboardNotificationHandler = blackboardNotificationHandler;

    }

    public void setLookUpServiceServiceLocator(ServiceLocator<ISLookUpService> lookUpServiceServiceLocator) {

        this.lookUpServiceServiceLocator = lookUpServiceServiceLocator;

    }

    public List<String> getFieldQueryRules() {

        return fieldQueryRules;

    }

    public void setFieldQueryRules(List<String> fieldQueryRules) {

        this.fieldQueryRules = fieldQueryRules;

    }

}

/**

 * 

 */

package eu.dnetlib.data.search.app.plan;

import org.apache.log4j.Logger;

/**

 * @author stoumpos

 */

public class PrefixRule extends QueryRewriteRule {

	public static final Logger logger = Logger.getLogger(QueryRewriteRule.class);

	private String prefix = null;

	public PrefixRule() {

		this(null, null);

	}

	public PrefixRule(String name, String prefix) {

		super(name);

		this.prefix = prefix;

	}

	public String getPrefix() {

		return prefix;

	}

	public void setPrefix(String prefix) {

		this.prefix = prefix;

	}

	/* (non-Javadoc)

	 * @see eu.dnetlib.data.search.app.plan.RewriteRule#apply(eu.dnetlib.data.search.app.plan.Query)

	 */

	public String apply(String query)  {

		String rewritten = query;

		StringBuffer sb = new StringBuffer(prefix);

		sb.append(" AND ").append(query);

		if (logger.isDebugEnabled()) {

			logger.debug("rewrite query '" + query

					+ "' as '" + sb.toString() + "'");

		}

       rewritten = sb.toString();

		return rewritten;

	}

}

package eu.dnetlib.data.search.web.utils;

import java.util.Arrays;

import java.util.List;

/**

 * Created by kiatrop on 19/9/2016.

 */

public class RequestResponseHandler {

    public enum Entity {

        RESULT("result"),

        PUBLICATION("publication"),

        DATASET("dataset"),

        PROJECT("project"),

        DATASOURCE("datasource"),

        ORGANIZATION("organization"),

        //PERSON("person"),

        SOFTWARE("software"),

        OTHER("other"),

        NONE("");

        private final List<String> RESULT_FIELD_QUERIES = Arrays.asList("oaftype exact result");

        private final List<String> PUBLICATION_FIELD_QUERIES = Arrays.asList("oaftype exact result", "resulttypeid exact publication");

        private final List<String> DATASET_FIELD_QUERIES = Arrays.asList("oaftype exact result", "resulttypeid exact dataset");

        private final List<String> SOFTWARE_FIELD_QUERIES = Arrays.asList("oaftype exact result", "resulttypeid exact software");

        private final List<String> OTHER_FIELD_QUERIES = Arrays.asList("oaftype exact result", "resulttypeid exact other");

        private final List<String> PROJECT_FIELD_QUERIES = Arrays.asList("oaftype exact project");

        private final List<String> DATASOURCE_FIELD_QUERIES = Arrays.asList("oaftype exact datasource", "(eosctype exact \"Data Source\")");

        private final List<String> ORGANIZATION_FIELD_QUERIES = Arrays.asList("oaftype exact organization");

        /*private final List<String> ORGANIZATION_FIELD_QUERIES = Arrays.asList("oaftype exact organization",

                "reldatasourcecompatibilityid exact driver or reldatasourcecompatibilityid exact driver-openaire2.0 or " +

                "reldatasourcecompatibilityid exact openaire2.0 or reldatasourcecompatibilityid exact openaire3.0 or " +

                "reldatasourcecompatibilityid exact openaire2.0_data or reldatasourcecompatibilityid exact hostedBy or relproject=*");

        */

        //private final List<String> PERSON_FIELD_QUERIES = Arrays.asList("oaftype exact person");

        private final String RESULT_QUERY = "oaftype exact result";

        private final String PUBLICATION_QUERY = "resulttypeid exact publication";

        private final String DATASET_QUERY = "resulttypeid exact dataset";

        private final String SOFTWARE_QUERY = "resulttypeid exact software";

        private final String OTHER_QUERY = "resulttypeid exact other";

        private String value;

        Entity(String value) {

            this.value = value;

        }

        private String getValue() {

            return value;

        }

        @Override

        public String toString() {

            return this.getValue();

        }

        public String getQueryPrefix() {

            switch (this) {

                case RESULT:

                    return RESULT_PREFIX;

                case PUBLICATION:

                    return PUBLICATION_PREFIX;

                case DATASET:

                    return DATASET_PREFIX;

                case SOFTWARE:

                    return SOFTWARE_PREFIX;

                case OTHER:

                    return OTHER_PREFIX;

                case PROJECT:

                    return PROJECT_PREFIX;

                case DATASOURCE:

                    return DATASOURCE_PREFIX;

                case ORGANIZATION:

                    return ORGANIZATION_PREFIX;

                //case PERSON:

                //    return PERSON_PREFIX;

                case NONE:

                    return "";

                default:

                    throw new IllegalArgumentException();

            }

        }

        public String getPlural() {

            switch (this) {

                case RESULT:

                    return "results";

                case PUBLICATION:

                    return "publications";

                case DATASET:

                    return "datasets";

                case SOFTWARE:

                    return "software";

                case OTHER:

                    return "other";

                case PROJECT:

                    return "projects";

                case DATASOURCE:

                    return "datasources";

                case ORGANIZATION:

                    return "organizations";

                //case PERSON:

                //    return "people";

                case NONE:

                    return "resources";

                default:

                    throw new IllegalArgumentException();

            }

        }

        public List<String> getFieldQueries() {

            switch (this) {

                case RESULT:

                    return RESULT_FIELD_QUERIES;

                case PUBLICATION:

                    return PUBLICATION_FIELD_QUERIES;

                case DATASET:

                    return DATASET_FIELD_QUERIES;

                case SOFTWARE:

                    return SOFTWARE_FIELD_QUERIES;

                case OTHER:

                    return OTHER_FIELD_QUERIES;

                case PROJECT:

                    return PROJECT_FIELD_QUERIES;

                case DATASOURCE:

                    return DATASOURCE_FIELD_QUERIES;

                case ORGANIZATION:

                    return ORGANIZATION_FIELD_QUERIES;

                //case PERSON:

                //    return PERSON_FIELD_QUERIES;

                default:

                    throw new IllegalArgumentException();

            }

        }

        public String getSimpleQuery() {

            switch (this) {

                case RESULT:

                    return RESULT_QUERY;

                case PUBLICATION:

                    return PUBLICATION_QUERY;

                case DATASET:

                    return DATASET_QUERY;

                case SOFTWARE:

                    return SOFTWARE_QUERY;

                case OTHER:

                    return OTHER_QUERY;

                default:

                    throw new IllegalArgumentException();

            }

        }

    }

    public final static String RESULT_PREFIX = "(oaftype exact result)";

    public final static String PUBLICATION_PREFIX = "(oaftype exact result) and (resulttypeid exact publication)";

    public final static String DATASET_PREFIX = "(oaftype exact result) and (resulttypeid exact dataset)";

    public final static String SOFTWARE_PREFIX = "(oaftype exact result) and (resulttypeid exact software)";

    public final static String OTHER_PREFIX = "(oaftype exact result) and (resulttypeid exact other)";

    public final static String PROJECT_PREFIX = "(oaftype exact project)";

    public final static String DATASOURCE_PREFIX = "(oaftype exact datasource) and (eosctype exact \"Data Source\")";

    public final static String ORGANIZATION_PREFIX = "(oaftype exact organization and " +

            "(reldatasourcecompatibilityid=driver or reldatasourcecompatibilityid=driver-openaire2.0 or " +

            "reldatasourcecompatibilityid=openaire2.0 or reldatasourcecompatibilityid=openaire3.0 or " +

            "reldatasourcecompatibilityid=openaire2.0_data or reldatasourcecompatibilityid=hostedBy " +

            "or relproject=*)";

    //public final static String PERSON_PREFIX = "(oaftype exact person)";

}

package eu.dnetlib.data.search.app.plan;

/**

 */

public abstract class FieldRewriteRule {

	protected String name = null;

	protected String fieldName = null;

	public FieldRewriteRule(String name, String fieldName) {

		this.name = name;

		this.fieldName = fieldName;

	}

	public String getFieldName() {

		return fieldName;

	}

	public void setFieldName(String fieldName) {

		this.fieldName = fieldName;

	}

	public String getName() {

		return name;

	}

	public void setName(String name) {

		this.name = name;

	}

	//public abstract CqlClause apply(CqlRelation relation) throws CqlException;

	@Override

	public String toString() {

		return name;

	}

}

package eu.dnetlib.data.search.solr;

import eu.dnetlib.api.DriverServiceException;

import eu.dnetlib.api.data.IndexService;

import eu.dnetlib.api.data.IndexServiceException;

import eu.dnetlib.domain.EPR;

import eu.dnetlib.domain.ServiceIdentity;

import eu.dnetlib.domain.data.Hint;

import eu.dnetlib.domain.enabling.Notification;

import java.util.List;

/**

 * Created by antleb on 2/4/14.

 */

public class SolrIndexClient implements IndexService {

    private String solrServerUrl = null;

    private String interpretation = null;

    @Override

    public EPR indexLookup(String id, String query, String mdformat, String layout) throws IndexServiceException {

        EPR epr = new EPR();

        epr.setAddress(solrServerUrl);

        epr.setParameter("action", "lookup");

        epr.setParameter("id", id);

        epr.setParameter("query", query);

        epr.setParameter("mdformat", mdformat);

        epr.setParameter("layout", layout);

        epr.setParameter("interpretation", this.interpretation);

        return epr;

    }

    @Override

    public EPR getBrowsingStatistics(String query, String index, String mdFormatId, String layoutId) throws IndexServiceException {

        EPR epr = new EPR();

        epr.setAddress(solrServerUrl);

        epr.setParameter("action", "browse");

        epr.setParameter("id", index);

        epr.setParameter("query", query);

        epr.setParameter("mdformat", mdFormatId);

        epr.setParameter("layout", layoutId);

        epr.setParameter("interpretation", this.interpretation);

        return epr;

    }

    public void newMethod(){}

    public EPR getBrowsingStatistics2(String query, String index, String mdFormatId, String layoutId) throws IndexServiceException {

        EPR epr = new EPR();

        epr.setAddress(solrServerUrl);

        epr.setParameter("action", "browse");

        epr.setParameter("id", index);

        epr.setParameter("query", query);

        epr.setParameter("mdformat", mdFormatId);

        epr.setParameter("layout", layoutId);

        epr.setParameter("interpretation", this.interpretation);

        return epr;

    }

    @Override

    public Hint suggestiveSearch(String s, String s2, String s3, String s4, String s5) throws IndexServiceException {

        throw new UnsupportedOperationException();

    }

    @Override

    public List<String> getListOfIndices() throws IndexServiceException {

        throw new UnsupportedOperationException();

    }

    @Override

    public String getIndexStatistics(String s) throws IndexServiceException {

        throw new UnsupportedOperationException();

    }

    @Override

    public String getListOfIndicesCSV() throws IndexServiceException {

        throw new UnsupportedOperationException();

    }

    @Override

    public ServiceIdentity identify() {

        throw new UnsupportedOperationException();

    }

    @Override

    public void notify(Notification notification) throws DriverServiceException {

        throw new UnsupportedOperationException();

    }

    public String getSolrServerUrl() {

        return solrServerUrl;

    }

    public void setSolrServerUrl(String solrServerUrl) {

        this.solrServerUrl = solrServerUrl;

    }

    public String getInterpretation() {

        return interpretation;

    }

    public void setInterpretation(String interpretation) {

        this.interpretation = interpretation;

    }

}

package eu.dnetlib.data.search.solr;

import eu.dnetlib.domain.EPR;

import eu.dnetlib.domain.SecureDriverResource;

import gr.uoa.di.driver.enabling.resultset.ResultSet;

import gr.uoa.di.driver.enabling.resultset.ResultSetFactory;

import io.micrometer.prometheus.PrometheusMeterRegistry;

import org.apache.log4j.Logger;

import org.apache.solr.client.solrj.impl.CloudSolrClient;

import org.springframework.beans.factory.annotation.Autowired;

import java.util.Arrays;

import java.util.HashMap;

import java.util.Map;

/**

 * Created by antleb on 2/4/14.

 */

public class SolrResultSetFactory implements ResultSetFactory {

    private Logger logger = Logger.getLogger(getClass());

    private Map<String, CloudSolrClient> clients = new HashMap<String, CloudSolrClient>();

    @Autowired

    private PrometheusMeterRegistry prometheusMeterRegistry;

    @Override

    public ResultSet<String> createResultSet(EPR epr) {

        try {

            String[] zkservers= epr.getAddress().split(",");

            CloudSolrClient solrClient;

            synchronized (clients) {

                solrClient =  clients.get(epr.getAddress());

            }

            if (solrClient == null) {

                solrClient = new CloudSolrClient.Builder().withZkHost(Arrays.asList(zkservers)).build();

                clients.put(epr.getAddress(), solrClient);

            }

            ResultSet<String> solrResultSets = new SolrResultSet(epr, solrClient, prometheusMeterRegistry);

            return solrResultSets;

        } catch (Exception e) {

            logger.warn("Error creating solr client", e);

        }

        logger.debug("NOT HERE!");

        return null;

    }

    @Override

    public <D> ResultSet<D> createResultSet(EPR epr, Class<D> resourceClass) {

        return (ResultSet<D>) createResultSet(epr);

    }

    @Override

    public <D> ResultSet<D> createResultSet(ResultSet<?> rs, Class<D> resourceClass) {

        throw new UnsupportedOperationException();

    }

    @Override

    public <D extends SecureDriverResource> ResultSet<D> createSecurityAwareRS(EPR epr, Class<D> resourceClass) {

        throw new UnsupportedOperationException();

    }

}

package eu.dnetlib.data.search.transform.config;