D-Net

<?xml version="1.0"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>uoa-search</artifactId>

	<packaging>jar</packaging>

	<version>3.13.4</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/uoa-search/tags/uoa-search-3.13.4</developerConnection> 

        </scm>

    <repositories>

        <!-- Cloudera Repositories -->

        <repository>

            <snapshots>

                <enabled>false</enabled>

            </snapshots>

            <id>cloudera-central</id>

            <name>cloundera-libs-release</name>

            <url>http://maven.research-infrastructures.eu/nexus/content/repositories/cloudera-central</url>

        </repository>

        <repository>

            <id>cloudera-snapshots</id>

            <name>cloudera-libs-snapshot</name>

            <url>http://maven.research-infrastructures.eu/nexus/content/repositories/cloudera-snapshots</url>

        </repository>

        <repository>

            <id>typesafe</id>

            <name>typesafe-releases</name>

            <url>http://maven.research-infrastructures.eu/nexus/content/repositories/typesafe</url>

        </repository>

    </repositories>

	<dependencies>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>uoa-commons</artifactId>

			<version>[2.0.0, 3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>org.mockito</groupId>

			<artifactId>mockito-all</artifactId>

			<version>1.6</version>

		</dependency>

		<dependency>

			<groupId>org.apache.solr</groupId>

			<artifactId>solr-solrj</artifactId>

			<version>7.2.1</version>

            <exclusions>

                <exclusion>

                    <groupId>org.apache.zookeeper</groupId>

                    <artifactId>zookeeper</artifactId>

                </exclusion>

            </exclusions>

		</dependency>

        <dependency>

            <groupId>org.apache.zookeeper</groupId>

            <artifactId>zookeeper</artifactId>

            <version>3.4.5-cdh4.3.0</version>

        </dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-cql-utils</artifactId>

			<version>[2.0.0, 3.0.0)</version>

			<scope>compile</scope>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-blackboard-common</artifactId>

			<version>[2.2.1, 3.0.0)</version>

		</dependency>

		<!-- Spring MVC framework -->

		<dependency>

			<groupId>org.springframework</groupId>

			<artifactId>spring-webmvc</artifactId>

			<version>${spring.version}</version>

		</dependency>

		<dependency>

			<groupId>org.apache.velocity</groupId>

			<artifactId>velocity</artifactId>

			<version>1.7</version>

		</dependency>

		<dependency>

			<groupId>asm</groupId>

			<artifactId>asm</artifactId>

			<version>3.3.1</version>

		</dependency>

		<dependency>

			<groupId>com.sun.jersey</groupId>

			<artifactId>jersey-bundle</artifactId>

			<version>1.19</version>

		</dependency>

		<dependency>

			<groupId>org.json</groupId>

			<artifactId>json</artifactId>

			<version>20140107</version>

		</dependency>

		<dependency>

			<groupId>com.sun.jersey</groupId>

			<artifactId>jersey-server</artifactId>

			<version>1.19</version>

		</dependency>

		<dependency>

			<groupId>com.sun.jersey</groupId>

			<artifactId>jersey-core</artifactId>

			<version>1.19</version>

		</dependency>

		<dependency>

			<groupId>commons-validator</groupId>

			<artifactId>commons-validator</artifactId>

			<version>1.6</version>

		</dependency>

		<dependency>

			<groupId>org.eclipse.persistence</groupId>

			<artifactId>org.eclipse.persistence.moxy</artifactId>

			<version>2.5.0</version>

		</dependency>

		<dependency>

			<groupId>com.sun.jersey.contribs</groupId>

			<artifactId>jersey-spring</artifactId>

			<version>1.8</version>

			<exclusions>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring-core</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring-web</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring-beans</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring-context</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring-utils</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring-aop</artifactId>

				</exclusion>

			</exclusions>

		</dependency>

		<!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->

		<dependency>

			<groupId>com.google.code.gson</groupId>

			<artifactId>gson</artifactId>

			<version>2.3.1</version>

		</dependency>

		<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->

		<dependency>

			<groupId>org.apache.commons</groupId>

			<artifactId>commons-lang3</artifactId>

			<version>3.5</version>

		</dependency>

		<dependency>

			<groupId>io.prometheus</groupId>

			<artifactId>simpleclient</artifactId>

			<version>0.6.0</version>

		</dependency>

		<dependency>

			<groupId>io.prometheus</groupId>

			<artifactId>simpleclient_servlet</artifactId>

			<version>0.6.0</version>

		</dependency>

		<dependency>

			<groupId>io.prometheus</groupId>

			<artifactId>simpleclient_hotspot</artifactId>

			<version>0.6.0</version>

		</dependency>

		<!-- https://mvnrepository.com/artifact/io.micrometer/micrometer-registry-prometheus -->

		<dependency>

			<groupId>io.micrometer</groupId>

			<artifactId>micrometer-registry-prometheus</artifactId>

			<version>1.2.0</version>

		</dependency>

		<!-- https://mvnrepository.com/artifact/io.micrometer/micrometer-spring-legacy -->

		<dependency>

			<groupId>io.micrometer</groupId>

			<artifactId>micrometer-spring-legacy</artifactId>

			<version>1.2.0</version>

		</dependency>

		<!-- https://mvnrepository.com/artifact/org.aspectj/aspectjrt -->

		<!-- AspectJ -->

		<dependency>

			<groupId>org.aspectj</groupId>

			<artifactId>aspectjrt</artifactId>

			<version>1.9.4</version>

		</dependency>

		<dependency>

			<groupId>org.aspectj</groupId>

			<artifactId>aspectjweaver</artifactId>

			<version>1.9.4</version>

		</dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.maven/maven-model -->

        <dependency>

            <groupId>org.apache.maven</groupId>

            <artifactId>maven-model</artifactId>

            <version>3.3.9</version>

        </dependency>

    </dependencies>

	<build>

		<plugins>

			<plugin>

				<groupId>org.apache.maven.plugins</groupId>

				<artifactId>maven-compiler-plugin</artifactId>

				<configuration>

					<source>1.8</source>

					<target>1.8</target>

				</configuration>

			</plugin>

		</plugins>

	</build>

</project>

package eu.dnetlib.data.search.web.api;

import io.micrometer.prometheus.PrometheusMeterRegistry;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.context.annotation.ComponentScan;

import org.springframework.http.MediaType;

import org.springframework.stereotype.Component;

import org.springframework.stereotype.Controller;

import org.springframework.web.bind.annotation.RequestMapping;

import org.springframework.web.bind.annotation.RequestMethod;

import org.springframework.web.bind.annotation.ResponseBody;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

@Controller

@ComponentScan

@Component

public class MicrometerPublisher {

    @Autowired

    PrometheusMeterRegistry registry;

    @RequestMapping(method = RequestMethod.GET, value = "/metrics", produces = {MediaType.TEXT_PLAIN_VALUE})

    @ResponseBody

    public String getMetrics(HttpServletRequest request, HttpServletResponse response) throws Exception {

        return  registry.scrape();

    }

}

package eu.dnetlib.data.search.web.api;

/**

 * Created by kiatrop on 20/12/2016.

 */

public class TSVResponseFormat {

    public static final String organization = "Name\tProjects\tCountry\n";

    public static final String response = "%s%s";

}

package eu.dnetlib.data.search.web.api;

/**

 * Created by kiatrop on 16/1/2017.

 */

public class HTMLResponseFormat {

    public static final String response = "<table>" +

            "<thead><tr>" +

            " <th>Title</th>" +

            "<th>Authors</th>" +

            "<th>Publication Year</th>" +

            "<th>DOI</th>\n" +

            "<th>Permanent Identifier</th>" +

            "<th>Publication type</th>" +

            "<th>Journal</th>" +

            "<th>Project Name (GA Number)</th>" +

            "<th>Access Mode</th>" +

            "</tr>" +

            "</thead>" +

            "<tbody>%s</tbody></table>";

}

package eu.dnetlib.data.search.micrometer;

import io.micrometer.core.instrument.Tags;

import io.micrometer.core.instrument.Timer;

import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics;

import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics;

import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics;

import io.micrometer.core.instrument.binder.system.UptimeMetrics;

import io.micrometer.prometheus.PrometheusMeterRegistry;

import org.apache.log4j.Logger;

import org.apache.maven.model.Dependency;

import org.apache.maven.model.Model;

import org.apache.maven.model.io.xpp3.MavenXpp3Reader;

import org.codehaus.plexus.util.xml.pull.XmlPullParserException;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.context.annotation.Configuration;

import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;

import javax.ws.rs.core.Application;

import java.io.IOException;

import java.io.InputStreamReader;

import java.time.Duration;

import java.util.List;

import java.util.concurrent.atomic.AtomicInteger;

@Configuration

@Component

public class Metrics {

    private static final Logger logger = Logger.getLogger(Metrics.class);

    private static final AtomicInteger n = new AtomicInteger(1);

    @Autowired

    private PrometheusMeterRegistry registry;

    @PostConstruct

    public void init() {

        new JvmMemoryMetrics().bindTo(registry);

        new JvmGcMetrics().bindTo(registry);

        new JvmThreadMetrics().bindTo(registry);

        new UptimeMetrics().bindTo(registry);

        Timer.builder("search.server.response.duration").description("search response time with solr request and transformation included")

                .sla(Duration.ofMillis(100), Duration.ofMillis(500), Duration.ofMillis(1000), Duration.ofMillis(2000), Duration.ofMillis(2500),

                        Duration.ofMillis(3000),Duration.ofMillis(4000), Duration.ofMillis(5000)).register(registry);

        Timer.builder("solr.server.response.duration").description("solr response time")

                .sla(Duration.ofMillis(100), Duration.ofMillis(500), Duration.ofMillis(1000), Duration.ofMillis(2000), Duration.ofMillis(2500),

                        Duration.ofMillis(3000),Duration.ofMillis(4000), Duration.ofMillis(5000)).register(registry);

        MavenXpp3Reader reader = new MavenXpp3Reader();

        Model model = null;

        try {

                model = reader.read(

                        new InputStreamReader(

                                Application.class.getResourceAsStream(

                                        "/META-INF/maven/eu.dnetlib/uoa-search/pom.xml"

                                )

                        )

                );

        } catch (XmlPullParserException e) {

           logger.error("Error reading pom. ", e);

        } catch (IOException e) {

            logger.error("Error reading pom. ", e);

        } {

        }

        List<Dependency> list = model.getDependencies();

        Dependency micrometer = list.stream()

                .filter(dependency -> dependency.getArtifactId().equals("micrometer-registry-prometheus"))

                .findFirst()

                .get();

        registry.gauge("micrometer_info", Tags.of("component", model.getArtifactId()).and("scmtag", "HEAD").and("version", model.getVersion()), n);

        //registry.summary("search.server.response.size");

    }

}

package eu.dnetlib.data.search.app.plan;

/**

 */

public abstract class QueryRewriteRule {

	protected String name = null;

	public QueryRewriteRule(String name) {

		this.name = name;

	}

	public QueryRewriteRule() {

		this("");

	}

	public String getName() {

		return name;

	}

	public void setName(String name) {

		this.name = name;

	}

	public abstract String apply(String query);

	@Override

	public String toString() {

		return "'"+name+"'";

	}

}

package eu.dnetlib.data.search.app.plan;

import eu.dnetlib.domain.functionality.Collection;

import gr.uoa.di.driver.enabling.ISLookUp;

import org.apache.log4j.Logger;

public class CollectionRewriteRule extends FieldRewriteRule {

	public static Logger logger = Logger.getLogger(QueryRewriteRule.class);

	private ISLookUp<Collection> collectionLookUp = null;

	public CollectionRewriteRule() {

		super(null, null);

	}

	public CollectionRewriteRule(String name, String fieldName) {

		super(name, fieldName);

	}

	public ISLookUp<Collection> getCollectionLookUp() {

		return collectionLookUp;

	}

	public void setCollectionLookUp(

			ISLookUp<eu.dnetlib.domain.functionality.Collection> collectionLookUp) {

		this.collectionLookUp = collectionLookUp;

	}

/*	@Override

	public CqlClause apply(CqlRelation relation) throws CqlException {

		if (!relation.getIndex().equals(fieldName)) {

			throw new IllegalArgumentException("Rule " + name

					+ " expects field " + fieldName

					+ " but is applied on field " + relation.getIndex());

		}

		String id = relation.getValue().substring(1, relation.getValue().length()-1);

		try {

			Collection collection = getCollectionLookUp().getById(id);

			if (logger.isDebugEnabled()) {

				logger.debug("rewrite relation '" + relation.toCqlString()

						+ "' as '" + collection.getRetrievalCondition() + "'");

			}

			Query query = new Query(collection.getRetrievalCondition());

			return query.getCqlQuery().getRoot();

		} catch (ISLookUpException e) {

			throw new CqlException(

					"Collection lookup for " + id + " failed.", e);

		}

	} */

}

package eu.dnetlib.data.search.solr;

import eu.dnetlib.api.data.SearchServiceException;

import eu.dnetlib.domain.EPR;

import eu.dnetlib.domain.SecureDriverResource;

import gr.uoa.di.driver.enabling.resultset.ResultSet;

import gr.uoa.di.driver.enabling.resultset.ResultSetFactory;

import io.micrometer.prometheus.PrometheusMeterRegistry;

import org.apache.log4j.Logger;

import org.apache.solr.client.solrj.impl.CloudSolrClient;

import org.springframework.beans.factory.annotation.Autowired;

import org.z3950.zing.cql.CQLParseException;

import java.io.IOException;

import java.util.Arrays;

import java.util.HashMap;

import java.util.Map;

/**

 * Created by antleb on 2/4/14.

 */

public class SolrResultSetFactory implements ResultSetFactory {

    private Logger logger = Logger.getLogger(getClass());

    //private Map<String, CloudSolrClient> clients = new HashMap<String, CloudSolrClient>();

    @Autowired

    private PrometheusMeterRegistry prometheusMeterRegistry;

    private CloudSolrClient solrClient;

    private CloudSolrClient solrClientWithSocketTimeout;

    public void init(String[] zkservers, int socketTimeout) {

        logger.info("Initialization of clients.");

        logger.info("solrClient is set without socket timeout.");

        solrClient = new CloudSolrClient.Builder().withZkHost(Arrays.asList(zkservers)).build();

        if (socketTimeout!=-1) {

            logger.info("solrClientWithSocketTimeout is set with socket timeout " + socketTimeout );

            solrClientWithSocketTimeout = new CloudSolrClient.Builder().withZkHost(Arrays.asList(zkservers))

                    .withSocketTimeout(socketTimeout).build();

        } else {  //sudo solrClientWithSocketTimeout

            logger.info("solrClientWithSocketTimeout has ΝΟ actual timeout as the socketTimeout property is set to -1.");

            solrClientWithSocketTimeout = new CloudSolrClient.Builder().withZkHost(Arrays.asList(zkservers)).build();

        }

    }

    public ResultSet<String> createResultSetWithSocketTimeout(EPR epr) throws IOException, CQLParseException, SearchServiceException {

        logger.info("Get RS with solrClient With SocketTimeout for EPR: " + epr);

        return new SolrResultSet(epr, solrClientWithSocketTimeout, prometheusMeterRegistry);

    }

    public ResultSet<String> createResultSetWithoutSocketTimeout(EPR epr) throws IOException, CQLParseException, SearchServiceException {

        logger.info("Get RS with solrClient Without SocketTimeout for EPR: " + epr);

        return new SolrResultSet(epr, solrClient, prometheusMeterRegistry);

    }

    @Override

    public ResultSet<String> createResultSet(EPR epr) {

        logger.info("Get RS with solrClient Without SocketTimeout for EPR: " + epr);

        try {

            return new SolrResultSet(epr, solrClient, prometheusMeterRegistry);

        } catch (IOException ioe) {

            logger.error("Error in search. I will return null resultSet.", ioe);

        } catch (CQLParseException cqlpe) {

            logger.error("Error in cql parse. I will return null resultSet.", cqlpe);

        } catch (SearchServiceException sse) {

            logger.error("Error in search.", sse);

        }

        return null;

    }

    @Override

    public <D> ResultSet<D> createResultSet(EPR epr, Class<D> resourceClass) {

        return (ResultSet<D>) createResultSet(epr);

    }

    @Override

    public <D> ResultSet<D> createResultSet(ResultSet<?> rs, Class<D> resourceClass) {

        throw new UnsupportedOperationException();

    }

    @Override

    public <D extends SecureDriverResource> ResultSet<D> createSecurityAwareRS(EPR epr, Class<D> resourceClass) {

        throw new UnsupportedOperationException();

    }

}

package eu.dnetlib.data.search.solr;

import com.google.gson.Gson;

import eu.dnetlib.api.data.SearchServiceException;

import eu.dnetlib.data.search.transform.Transformer;

import eu.dnetlib.data.search.transform.TransformerException;

import eu.dnetlib.data.search.utils.solr.SolrResultSetOptionsUtil;

import eu.dnetlib.data.search.utils.solr.SolrResultsFormatter;

import eu.dnetlib.domain.EPR;

import gr.uoa.di.driver.enabling.resultset.ResultSet;

import io.micrometer.core.instrument.Timer;

import io.micrometer.prometheus.PrometheusMeterRegistry;

import org.apache.commons.lang.StringEscapeUtils;

import org.apache.log4j.Logger;

import org.apache.solr.client.solrj.SolrClient;

import org.apache.solr.client.solrj.SolrServerException;

import org.apache.solr.client.solrj.impl.CloudSolrClient;

import org.apache.solr.client.solrj.response.FacetField;

import org.apache.solr.client.solrj.response.QueryResponse;

import org.apache.solr.common.SolrDocumentList;

import org.apache.solr.common.params.SolrParams;

import org.apache.solr.common.util.NamedList;

import org.z3950.zing.cql.CQLParseException;

import javax.ws.rs.core.MediaType;

import java.io.IOException;

import java.io.OutputStream;

import java.util.*;

/**

 * Created by antleb on 2/4/14.

 */

public class SolrResultSet implements ResultSet<String> {

    private Logger logger = Logger.getLogger(getClass());

    private EPR epr = null;

    public SolrClient solrClient = null;

    private NamedList<String> queryOpts = new NamedList<String>();

    long size = -1;

    private PrometheusMeterRegistry registry;

    public SolrResultSet(EPR epr, CloudSolrClient cloudSolrClient, PrometheusMeterRegistry registry)

            throws IOException, CQLParseException, SearchServiceException {

        this.epr = epr;

        this.solrClient = cloudSolrClient;

        this.queryOpts = SolrResultSetOptionsUtil.extractQueryOptions(epr.getParameter("query"));

        String layout = epr.getParameter("layout");

        String mdformat = epr.getParameter("mdformat");

        String interpretation = epr.getParameter("interpretation");

        ((CloudSolrClient)solrClient).setDefaultCollection(mdformat + "-" + layout + "-" + interpretation);

        this.registry = registry;

    }

    @Override

    public boolean isOpen() {

        return true;

    }

    @Override

    public boolean isAlive() {

        return true;

    }

    @Override

    public void close() {

        /* Closes the whole pull. We do not want this.

        try {

            logger.info("!!!!!!!!! !!!!!!! CLOSING !!!!!!!!! !!!!!!!!!! ");

            solrClient.close();

        } catch (IOException e) {

            logger.error("Error closing result set.", e);

        }*/

    }

    @Override

    public int size() {

        return (int) size;

    }

    @Override

    @Deprecated

    public List<String> getElements(int from, int to) {

        return get(from, to);

    }

    List<FacetField> facetFields = null;

    @Override

    @Deprecated

    public List<String> get(int from, int to) {

        List<String> res = new ArrayList<String>();

        QueryResponse rsp = null;

        HashMap<String, List<String>> map = new HashMap<String, List<String>>();

        logger.debug("from: " + from);

        logger.debug("to: " + to);

        queryOpts.add("start", from+1 + "");

        queryOpts.add("rows", to + 1+"");

        try {

            rsp = solrClient.query(SolrParams.toSolrParams(queryOpts));

            facetFields = rsp.getFacetFields();

            SolrDocumentList docs = rsp.getResults();

            if (facetFields!=null && !facetFields.isEmpty()) {

                for (int i = from - 1; i < to; i++) {

                    for (FacetField field : facetFields) {

                        if (field.getValueCount() > i) {

                            BrowseField bf = new BrowseField();

                            bf.setId(field.getValues().get(i).getName());

                            bf.setName(field.getValues().get(i).getName());

                            bf.setCount(field.getValues().get(i).getCount() + "");

                            if (map.get(field.getName()) == null) {

                                map.put(field.getName(), new ArrayList<String>());

                            }

                            map.get(field.getName()).add(new Gson().toJson(bf));

                        }

                    }

                }

                for (Map.Entry<String, List<String>> facetEntry : map.entrySet()) {

                    StringBuilder builder = new StringBuilder();

                    builder.append("\"" + facetEntry.getKey() + "\"" + " : ");

                    builder.append(facetEntry.getValue());

                    res.add(builder.toString());

                }

            }

            logger.debug("time: " + rsp.getElapsedTime());

            logger.debug("found: " + docs.getNumFound());

            logger.debug("docs: " + docs.size());

            for (int i = 0; i < docs.size(); i++) {

                String result = (String) docs.get(i).get("__result");

                res.add(result);

            }

            return res;

        } catch (SolrServerException sse) {

            logger.error("Fail to get results from Solr. ", sse);

        } catch (IOException ioe) {

            logger.error("Fail to get results from Solr. ", ioe);

        }

        return null;

    }

    @Override

    public EPR getEpr() {

        return null;

    }

    public Map<String,List<String>> newGet(int from, int to, String format, Transformer transformer,

                                           Transformer oldRefineTransformer, boolean timeout, String timeoutValue) throws SearchServiceException {

        List<String> refineSolrResults = new ArrayList<String>();

        List<String> searchSolrResults = new ArrayList<String>();

        QueryResponse rsp = null;

        HashMap<String, List<String>> map = new HashMap<String, List<String>>();

        if (from < 0) from = 0;

        if (to < 0) to = 0;

        queryOpts.add("start", from*to + "");

        queryOpts.add("rows", to +"");

        try {

            if (timeout) {

                if (timeoutValue != null && Integer.valueOf(timeoutValue) == -1) {

                    logger.warn("Timeout value is -1. Timeout will not be set for query: " + queryOpts.get("q"));

                    timeout = false;

                } else if (Integer.valueOf(timeoutValue) < 500) {

                    logger.warn("Timeout is less than 500ms. We fall back to the default 1000ms for query:" + queryOpts.get("q"));

                    queryOpts.add("timeAllowed", "500");

                } else if (timeout) {

                    queryOpts.add("timeAllowed", timeoutValue);

                }

            }

        } catch (NumberFormatException nfe) {

            logger.warn("NumberFormatException occurred. Timeout not properly set. We fall back to the default 1000ms.");

            queryOpts.add("timeAllowed", "500");

        }

        logger.info("timeout: " + timeout + ", timeout value: " + queryOpts.get("timeAllowed"));

        //queryOpts.add("f.resulthostingdatasource.facet.limit", "2");

        long startTime = System.nanoTime();

        try {

            io.micrometer.core.instrument.Timer.Sample timer = Timer.start(registry);

            rsp = solrClient.query(SolrParams.toSolrParams(queryOpts));

            timer.stop(registry.timer("solr.server.response.duration"));

            long estimatedTime = System.nanoTime() - startTime;

            logger.info("Solrj time " + estimatedTime/1000000 +  " milliseconds for query:" + queryOpts.get("q") +

                    " and facets " + queryOpts.getAll("facet.field") + " and fq " + queryOpts.getAll("fq") +

                    " and sorts " + queryOpts.getAll("sort") + " from: "

                    + from + " and size " + to + " with timeout: " + timeout);

            /*System.out.println("Solrj time " + estimatedTime/1000000 +  " milliseconds for query:" + queryOpts.get("q") +

                    " and facets " + queryOpts.getAll("facet.field") + " and fq " + queryOpts.getAll("fq") +

                    " and sorts " + queryOpts.getAll("sort") + " from: "

                    + from + " and size " + to);

            */

            facetFields = rsp.getFacetFields();

            SolrDocumentList docs = rsp.getResults();

            this.size = docs.getNumFound();

            if (facetFields!=null && !facetFields.isEmpty()) {

                if (format != null && format.equals(MediaType.APPLICATION_JSON)) {

                    for (FacetField field : facetFields) {

                        map.put(field.getName(), new ArrayList<String>());

                        BrowseField bf = null;

                        for (int i = 0; i < field.getValueCount(); i++) {

                            bf = new BrowseField();

                            //bf.setId(org.apache.commons.lang3.StringEscapeUtils.escapeJson(field.getValues().get(i).getName()));

                            bf.setId(field.getValues().get(i).getName());

                            String[] facetedValues = field.getValues().get(i).getName().split("\\|\\|",2);

                            logger.debug("faceted values " + Arrays.toString(facetedValues));

                            if (facetedValues.length > 1) {

                                //bf.setName(org.apache.commons.lang3.StringEscapeUtils.escapeJson(facetedValues[1]));

                                bf.setName(facetedValues[1]);

                                logger.debug("faceted values [1] " + facetedValues[1]);

                            } else if (field.getValues().get(i).getName().split("_\\:\\:",2).length > 1) {

                                //bf.setName(org.apache.commons.lang3.StringEscapeUtils.escapeJson(field.getValues().get(i).getName().split("\\:\\:",2)[1]).replaceAll("\\:\\:", "\\|"));

                                bf.setName(field.getValues().get(i).getName().split("\\:\\:",2)[1].replaceAll("\\:\\:", "\\|"));

                            } else {

                                //bf.setName(org.apache.commons.lang3.StringEscapeUtils.escapeJson(field.getValues().get(i).getName()));

                                bf.setName(field.getValues().get(i).getName());

                            }

                            bf.setCount(field.getValues().get(i).getCount() + "");

                            map.get(field.getName()).add(new Gson().toJson(bf));

                        }

                    }

                    StringBuilder builder = null;

                    for (Map.Entry<String, List<String>> facetEntry : map.entrySet()) {

                        builder = new StringBuilder();

                        builder.append("\"" + facetEntry.getKey() + "\"" + " : ");

                        builder.append(facetEntry.getValue());

                        refineSolrResults.add(builder.toString());

                    }

                } else { //the old implementation & xml as default

                    logger.debug("Creating old browse results.");

                    createXmlRefineFields(refineSolrResults, oldRefineTransformer);

                }

            }

            startTime = System.nanoTime();

            for (int i = 0; i < docs.size(); i++) {

                String result = (String) docs.get(i).get("__result");

                //logger.debug("["+ i +"]: " + docs.get(i).get("__result"));

                try {

                    if (transformer != null) {

                        //logger.debug("1 >>>>>>" + result);

                        String xml = result.replaceAll("<em>","").replaceAll("</em>","");

                        result = transformer.transform(xml);

                        //logger.debug("2 >>>>>>" + result);

                    }

                } catch (TransformerException te) {

                    logger.warn("Error transforming " + result, te);

                    continue;

                }

                if (format != null && format.equals(MediaType.APPLICATION_JSON)) {

                    searchSolrResults.add(SolrResultsFormatter.xml2Json(result));

                } else { // default xml

                    searchSolrResults.add(result);

                }

            }

            estimatedTime = System.nanoTime() - startTime;

            logger.info("Internal transformation time " + estimatedTime/1000000 +  " milliseconds for query:" + queryOpts.get("q") +

                    " and facets " + queryOpts.getAll("facet.field") + " and fq " + queryOpts.getAll("fq") + " from: "

                    + from + " and size " + to);

            Map<String,List<String>> response = new HashMap<String, List<String>>();

            //logger.debug("refine results " + refineSolrResults);

            //logger.info("search results SIZE " + searchSolrResults.size());

            //logger.info("search results " + searchSolrResults);

            response.put("refine",refineSolrResults);

            response.put("search", searchSolrResults);

            return response;

        } catch (SolrServerException sse) {

            logger.error("Error calling Solr for  query:" + queryOpts.get("q") + " and facets "

                    + queryOpts.getAll("facet.field") + " and fq " + queryOpts.getAll("fq") + " from: " + from +

                    " and size " + to, sse);

            throw new SearchServiceException("Error contacting Solr.", sse);

        } catch (IOException ioe) {

            logger.error("Error calling Solr for  query:" + queryOpts.get("q") + " and facets "

                    + queryOpts.getAll("facet.field") + " and fq " + queryOpts.getAll("fq") + " from: " + from +

                    " and size " + to, ioe);

            throw new SearchServiceException("Error reading results from Solr.", ioe);

        }

    }

    /**

     * limit is the maximum number of results the cursor get is allowed to fetch. If limit is set to -1 all

     * results are returned.

     */

    public void cursorGet(Transformer transformer, int limit, OutputStream os) throws SolrServerException, SearchServiceException {

        int rows = 500;

        int limitCounter = -1;

        queryOpts.add("start", "0");

        queryOpts.add("rows", "0");

        queryOpts.remove("rows");

        queryOpts.add("rows", rows+"");

        queryOpts.add("fl", "__result");

        queryOpts.add("shards.tolerant","true");

        queryOpts.add("cursorMark", "*");

        queryOpts.add("sort", "__indexrecordidentifier asc");

        String cursorMark = "*";

        String nextCursorMark = "";

        if ( limit > 0 ) {

            limitCounter = limit/rows;

            logger.info("limit counter " + limitCounter);

        }

        try {

            QueryResponse resp = solrClient.query(SolrParams.toSolrParams(queryOpts));

            while (!cursorMark.equals(nextCursorMark) && ( limitCounter > 0 || limitCounter == -1)) {

                //logger.info(">> " + limitCounter);

                resp = solrClient.query(SolrParams.toSolrParams(queryOpts));

                cursorMark = nextCursorMark;

                nextCursorMark = resp.getNextCursorMark();

                for (int i = 0; i < resp.getResults().size(); i++) {

                    if (transformer != null) {

                        String result = null;

                        try {

                            logger.debug("PRE RESULT " + resp.getResults().get(i).get("__result"));

                            result = transformer.transform((String) resp.getResults().get(i).get("__result"));

                            logger.debug("RESULT " + result);

                        } catch (TransformerException te) {

                            logger.warn("Error transforming " + result, te);

                            continue;

                        }

                        try {

                            os.write(result.getBytes());

                            os.flush();

                        } catch (IOException e) {

                            logger.error("Cursor get... ", e);

                            continue;

                        }

                    }

                }

                queryOpts.remove("cursorMark");

                queryOpts.add("cursorMark", nextCursorMark);

                limitCounter--;

            }

        } catch (IOException ioe) {

            logger.error("Error executing solr query. ", ioe);

        }

    }

    //TODO get rid of this as soon as Joomla portal is out

    //Just copied and refactored the old one...

    @Deprecated

    private void createXmlRefineFields(List<String> res, Transformer oldRefineTransformer) {

        int max = -12;

        for (FacetField field:facetFields) {

            logger.debug("field " + field.getName() + " has count " + field.getValueCount());

            if (field.getValueCount() > max) {

                max = field.getValueCount();

            }

        }

        logger.debug("max " + max);

        for (int i = 0; i < max; i++) {

            StringBuilder sb = new StringBuilder();

            sb.append("<row>");

            for (FacetField field:facetFields) {

                if (field.getValueCount() > i) {

                    sb.append("<groupresult field=\"").append(field.getName()).append("\">");

                    sb.append("<count>").append(field.getValues().get(i).getCount()).append("</count>");

                    sb.append("<originalValue>").append(StringEscapeUtils.escapeXml(field.getValues().get(i).getName())).append("</originalValue>");

                    String[] facetValues = field.getValues().get(i).getName().split("\\|\\|");

                    if(facetValues.length > 1) {

                        sb.append("<value>").append(StringEscapeUtils.escapeXml(facetValues[1])).append("</value>");

                    } else {

                        sb.append("<value>").append(StringEscapeUtils.escapeXml(facetValues[0])).append("</value>");

                    }

                    sb.append("</groupresult>");

                }

            }

            sb.append("</row>");

            try {

                //logger.debug("row: " + sb.toString());

                //logger.debug("row2: " + oldRefineTransformer.transform(sb.toString()));

                //TODO remove

                res.add(oldRefineTransformer.transform(sb.toString()));

            } catch (TransformerException te) {

                logger.error("Cannot transform refine for: " + sb.toString(), te);

            }

        }

    }

 /*   public static void main(String[] args) throws IOException, CQLParseException, SolrServerException {

        CloudSolrServer solrClient = new CloudSolrServer("beta.solr.openaire.eu:9983");

        solrClient.setDefaultCollection("DMF-index-openaire");

        NamedList<String> queryOpts = new NamedList<String>();

        //q=*:*&start=0&rows=10&cursorMark=*&sort=dateofcollection asc

        queryOpts.add("q", new CqlTranslatorImpl().getTranslatedQuery("objIdentifier = acm_________::0002c24f82c295e925a2bdf7bbf49bfc").asLucene());

        queryOpts.add("start", "0");

        queryOpts.add("rows", "1");

        queryOpts.add("fl", "__result");

        queryOpts.add("shards.tolerant","true");

        queryOpts.add("cursorMark", "*");

        queryOpts.add("sort", "__indexrecordidentifier asc");

        //queryOpts.add("q", new CqlTranslatorImpl().getTranslatedQuery("oaftype exact project").asLucene());

        NamedList<String> extraOpts = new NamedList<String>();

        QueryResponse resp = solrClient.query(SolrParams.toSolrParams(queryOpts));

        System.out.println("results " + resp.getResults().size());

        String cursorMark = "*";

        String nextCursorMark = "";

        int curs = 0;

        while (!cursorMark.equals(nextCursorMark)) {

            System.out.println("cursor " + cursorMark);

            System.out.println("next cursor " + nextCursorMark);

            cursorMark = nextCursorMark;

            for (int i = 0; i < resp.getResults().size(); i++) {

                String result = ((ArrayList<String>) resp.getResults().get(i).get("__result")).get(0);

                //System.out.println(result);

                resp = solrClient.query(SolrParams.toSolrParams(queryOpts));

            }

            nextCursorMark = resp.getNextCursorMark();

            queryOpts.add("cursorMark", nextCursorMark);

            System.out.println("CURS " + curs);

            curs ++;

        }

        //System.out.println((new CqlTranslatorImpl().getTranslatedQuery("objIdentifier = acm_________::0002c24f82c295e925a2bdf7bbf49bfc").asLucene()));

        //extraOpts.add("start", "1");

       // extraOpts.add("rows", "10");

       // extraOpts.addAll(queryOpts);

        //queryOpts.add("facet", "true");

        //TranslatedQuery translatedQuery = new CqlTranslatorImpl().getTranslatedQuery("oaftype=result sortBy resultdateofacceptance/sort.descending");

     //   queryOpts.add("q", "oaftype=project");

        //queryOpts.add("facet", "true");

        //queryOpts.add("facet.mincount", "1");

        //queryOpts.add("fq", "popularity");

//        queryOpts.put("fq", new CqlTranslatorImpl().getTranslatedQuery("").asLucene());

       // queryOpts.add("facet.field", "contextid");

       //  queryOpts.add("facet.field", "contextname");

       //  queryOpts.add("facet.mincount", "1");

       //  queryOpts.add("facet.threads", "10");

       // System.out.println(translatedQuery.getOptions().getSort().getMode());

       // System.out.println(translatedQuery.getOptions().getSort().getField());

        //queryOpts.add("sort", translatedQuery.getOptions().getSort().getField() + " " + translatedQuery.getOptions().getSort().getMode() );

/*        QueryResponse resp = null;

        synchronized (solrClient) {

            resp = solrClient.query(SolrParams.toSolrParams(extraOpts));

        }*/

//        System.out.println("time: " + resp.getElapsedTime());

    //System.out.println("results: " + resp.getResults());

/*      System.out.println(resp.getFacetField("contextname").getValueCount());

        for (FacetField.Count count:resp.getFacetField("contextname").getValues())

            System.out.println(count.getName() + " : " +  count.getCount());

        int max = -12;

        for (FacetField field:resp.getFacetFields()) {

            if (field.getValueCount() > max)

                max = field.getValueCount();

        }

        System.out.println("max: " + max);

*/

 //   }

//    @Override

//    public EPR getEpr() {

//        return epr;

//   }

}

class BrowseField {

    String name;

    public String getName() {

        return name;

    }

    public void setName(String name) {

        this.name = name;

    }

    public String getId() {

        return id;

    }

    public void setId(String id) {

        this.id = id;

    }

    public String getCount() {

        return count;

    }

    public void setCount(String count) {

        this.count = count;

    }

    String id;

    String count;

}

package eu.dnetlib.data.search.solr;

import eu.dnetlib.api.data.IndexService;

import eu.dnetlib.api.enabling.ISLookUpService;

import eu.dnetlib.api.enabling.ISLookUpServiceException;

import gr.uoa.di.driver.util.ServiceLocator;

import org.apache.log4j.Logger;

import java.util.List;

/**

 * Created by antleb on 2/4/14.

 */

public class SolrClientLocator implements ServiceLocator<IndexService> {

    private Logger logger = Logger.getLogger(getClass());

    private SolrIndexClient indexClient = null;

    private ServiceLocator<ISLookUpService> lookUpServiceServiceLocator = null;

    private String interpretation = null;

    private String cachedUrl = null;

    @Override

    public IndexService getService() {

        if (indexClient == null) {

            indexClient = locateIndexService();

        }

        return indexClient;

    }

    private SolrIndexClient locateIndexService() {

        SolrIndexClient client = new SolrIndexClient();

        try {

            List<String> urls = lookUpServiceServiceLocator.getService().quickSearchProfile("for $x in //RESOURCE_PROFILE//PROTOCOL[./@name='solr'] return data($x/@address)");

            if (urls.size() > 0) {

                logger.debug("cached url " + cachedUrl);

                cachedUrl = chooseIndexService(urls);

            } else {

                cachedUrl = getCachedIndexService(urls);

                logger.warn("Falling back to cached value for index : " + cachedUrl);

                throw new IllegalArgumentException("Could not locate an index service with a SOLR interface");

            }

        } catch (ISLookUpServiceException ise) {

            logger.warn("Error locating service", ise);

        } finally {

            logger.warn("Falling back to cached value for index : " + cachedUrl);

            client.setSolrServerUrl(cachedUrl);

            client.setInterpretation(this.interpretation);

        }

        return client;

    }

    private String getCachedIndexService(List<String> urls) {

        if (cachedUrl.equals("${services.index.default.url}"))

            return urls.get(0);

        return cachedUrl;

    }

    private String chooseIndexService(List<String> urls) {

        return urls.get(0);

    }

    public ServiceLocator<ISLookUpService> getLookUpServiceServiceLocator() {

        return lookUpServiceServiceLocator;

    }

    public void setLookUpServiceServiceLocator(ServiceLocator<ISLookUpService> lookUpServiceServiceLocator) {

        this.lookUpServiceServiceLocator = lookUpServiceServiceLocator;

    }

    public String getInterpretation() {

        return interpretation;

    }

    public void setInterpretation(String interpretation) {

        this.interpretation = interpretation;

    }

    public String getCachedUrl() {

        return cachedUrl;

    }

    public void setCachedUrl(String cachedUrl) {

        this.cachedUrl = cachedUrl;

    }

}

package eu.dnetlib.data.search.solr;

import eu.dnetlib.api.DriverServiceException;

import eu.dnetlib.api.data.IndexService;

import eu.dnetlib.api.data.IndexServiceException;

import eu.dnetlib.domain.EPR;

import eu.dnetlib.domain.ServiceIdentity;

import eu.dnetlib.domain.data.Hint;

import eu.dnetlib.domain.enabling.Notification;

import java.util.List;

/**

 * Created by antleb on 2/4/14.

 */

public class SolrIndexClient implements IndexService {

    private String solrServerUrl = null;

    private String interpretation = null;

    @Override

    public EPR indexLookup(String id, String query, String mdformat, String layout) throws IndexServiceException {

        EPR epr = new EPR();

        epr.setAddress(solrServerUrl);

        epr.setParameter("action", "lookup");

        epr.setParameter("id", id);

        epr.setParameter("query", query);

        epr.setParameter("mdformat", mdformat);

        epr.setParameter("layout", layout);

        epr.setParameter("interpretation", this.interpretation);

        return epr;

    }

    @Override

    public EPR getBrowsingStatistics(String query, String index, String mdFormatId, String layoutId) throws IndexServiceException {

        EPR epr = new EPR();

        epr.setAddress(solrServerUrl);

        epr.setParameter("action", "browse");

        epr.setParameter("id", index);

        epr.setParameter("query", query);

        epr.setParameter("mdformat", mdFormatId);

        epr.setParameter("layout", layoutId);

        epr.setParameter("interpretation", this.interpretation);

        return epr;

    }

    public void newMethod(){}

    public EPR getBrowsingStatistics2(String query, String index, String mdFormatId, String layoutId) throws IndexServiceException {

        EPR epr = new EPR();

        epr.setAddress(solrServerUrl);

        epr.setParameter("action", "browse");

        epr.setParameter("id", index);

        epr.setParameter("query", query);

        epr.setParameter("mdformat", mdFormatId);

        epr.setParameter("layout", layoutId);

        epr.setParameter("interpretation", this.interpretation);

        return epr;

    }

    @Override

    public Hint suggestiveSearch(String s, String s2, String s3, String s4, String s5) throws IndexServiceException {

        throw new UnsupportedOperationException();

    }

    @Override

    public List<String> getListOfIndices() throws IndexServiceException {

        throw new UnsupportedOperationException();

    }

    @Override

    public String getIndexStatistics(String s) throws IndexServiceException {

        throw new UnsupportedOperationException();

    }

    @Override

    public String getListOfIndicesCSV() throws IndexServiceException {

        throw new UnsupportedOperationException();

    }

    @Override

    public ServiceIdentity identify() {

        throw new UnsupportedOperationException();

    }

    @Override

    public void notify(Notification notification) throws DriverServiceException {

        throw new UnsupportedOperationException();

    }

    public String getSolrServerUrl() {

        return solrServerUrl;

    }

    public void setSolrServerUrl(String solrServerUrl) {

        this.solrServerUrl = solrServerUrl;

    }

    public String getInterpretation() {

        return interpretation;

    }

    public void setInterpretation(String interpretation) {

        this.interpretation = interpretation;

    }

}

package eu.dnetlib.data.search.app.plan;

/**

 */

public abstract class FieldRewriteRule {

	protected String name = null;

	protected String fieldName = null;

	public FieldRewriteRule(String name, String fieldName) {

		this.name = name;

		this.fieldName = fieldName;

	}

	public String getFieldName() {

		return fieldName;

	}

	public void setFieldName(String fieldName) {

		this.fieldName = fieldName;

	}

	public String getName() {

		return name;

	}

	public void setName(String name) {

		this.name = name;

	}

	//public abstract CqlClause apply(CqlRelation relation) throws CqlException;

	@Override

	public String toString() {

		return name;

	}

}

/**

 * 

 */

package eu.dnetlib.data.search.web;

import eu.dnetlib.api.data.SearchService;

import gr.uoa.di.driver.enabling.resultset.ResultSetFactory;

import org.springframework.context.ApplicationContext;

import org.springframework.web.context.support.WebApplicationContextUtils;

import org.w3c.dom.Document;

import org.w3c.dom.Element;

import org.xml.sax.InputSource;

import org.xml.sax.SAXException;

import javax.servlet.ServletContext;

import javax.servlet.http.HttpServletRequest;

import javax.xml.parsers.DocumentBuilder;