D-Net

package eu.dnetlib.data.search.app.plan;

/**

 */

public abstract class QueryRewriteRule {

	protected String name = null;

	public QueryRewriteRule(String name) {

		this.name = name;

	}

	public QueryRewriteRule() {

		this("");

	}

	public String getName() {

		return name;

	}

	public void setName(String name) {

		this.name = name;

	}

	public abstract String apply(String query);

	@Override

	public String toString() {

		return "'"+name+"'";

	}

}

package eu.dnetlib.data.search.app.plan;

import eu.dnetlib.domain.functionality.Collection;

import gr.uoa.di.driver.enabling.ISLookUp;

import org.apache.log4j.Logger;

public class CollectionRewriteRule extends FieldRewriteRule {

	public static Logger logger = Logger.getLogger(QueryRewriteRule.class);

	private ISLookUp<Collection> collectionLookUp = null;

	public CollectionRewriteRule() {

		super(null, null);

	}

	public CollectionRewriteRule(String name, String fieldName) {

		super(name, fieldName);

	}

	public ISLookUp<Collection> getCollectionLookUp() {

		return collectionLookUp;

	}

	public void setCollectionLookUp(

			ISLookUp<eu.dnetlib.domain.functionality.Collection> collectionLookUp) {

		this.collectionLookUp = collectionLookUp;

	}

/*	@Override

	public CqlClause apply(CqlRelation relation) throws CqlException {

		if (!relation.getIndex().equals(fieldName)) {

			throw new IllegalArgumentException("Rule " + name

					+ " expects field " + fieldName

					+ " but is applied on field " + relation.getIndex());

		}

		String id = relation.getValue().substring(1, relation.getValue().length()-1);

		try {

			Collection collection = getCollectionLookUp().getById(id);

			if (logger.isDebugEnabled()) {

				logger.debug("rewrite relation '" + relation.toCqlString()

						+ "' as '" + collection.getRetrievalCondition() + "'");

			}

			Query query = new Query(collection.getRetrievalCondition());

			return query.getCqlQuery().getRoot();

		} catch (ISLookUpException e) {

			throw new CqlException(

					"Collection lookup for " + id + " failed.", e);

		}

	} */

}

<?xml version="1.0"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>uoa-search</artifactId>

	<packaging>jar</packaging>

	<version>3.13.5-SNAPSHOT</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/uoa-search/branches/uoa-search-pre-3.9.4</developerConnection> 

        </scm>

    <repositories>

        <!-- Cloudera Repositories -->

        <repository>

            <snapshots>

                <enabled>false</enabled>

            </snapshots>

            <id>cloudera-central</id>

            <name>cloundera-libs-release</name>

            <url>http://maven.research-infrastructures.eu/nexus/content/repositories/cloudera-central</url>

        </repository>

        <repository>

            <id>cloudera-snapshots</id>

            <name>cloudera-libs-snapshot</name>

            <url>http://maven.research-infrastructures.eu/nexus/content/repositories/cloudera-snapshots</url>

        </repository>

        <repository>

            <id>typesafe</id>

            <name>typesafe-releases</name>

            <url>http://maven.research-infrastructures.eu/nexus/content/repositories/typesafe</url>

        </repository>

    </repositories>

	<dependencies>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>uoa-commons</artifactId>

			<version>[2.0.0, 3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>org.mockito</groupId>

			<artifactId>mockito-all</artifactId>

			<version>1.6</version>

		</dependency>

		<dependency>

			<groupId>org.apache.solr</groupId>

			<artifactId>solr-solrj</artifactId>

			<version>7.2.1</version>

            <exclusions>

                <exclusion>

                    <groupId>org.apache.zookeeper</groupId>

                    <artifactId>zookeeper</artifactId>

                </exclusion>

            </exclusions>

		</dependency>

        <dependency>

            <groupId>org.apache.zookeeper</groupId>

            <artifactId>zookeeper</artifactId>

            <version>3.4.5-cdh4.3.0</version>

        </dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-cql-utils</artifactId>

			<version>[2.0.0, 3.0.0)</version>

			<scope>compile</scope>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-blackboard-common</artifactId>

			<version>[2.2.1, 3.0.0)</version>

		</dependency>

		<!-- Spring MVC framework -->

		<dependency>

			<groupId>org.springframework</groupId>

			<artifactId>spring-webmvc</artifactId>

			<version>${spring.version}</version>

		</dependency>

		<dependency>

			<groupId>org.apache.velocity</groupId>

			<artifactId>velocity</artifactId>

			<version>1.7</version>

		</dependency>

		<dependency>

			<groupId>asm</groupId>

			<artifactId>asm</artifactId>

			<version>3.3.1</version>

		</dependency>

		<dependency>

			<groupId>com.sun.jersey</groupId>

			<artifactId>jersey-bundle</artifactId>

			<version>1.19</version>

		</dependency>

		<dependency>

			<groupId>org.json</groupId>

			<artifactId>json</artifactId>

			<version>20140107</version>

		</dependency>

		<dependency>

			<groupId>com.sun.jersey</groupId>

			<artifactId>jersey-server</artifactId>

			<version>1.19</version>

		</dependency>

		<dependency>

			<groupId>com.sun.jersey</groupId>

			<artifactId>jersey-core</artifactId>

			<version>1.19</version>

		</dependency>

		<dependency>

			<groupId>commons-validator</groupId>

			<artifactId>commons-validator</artifactId>

			<version>1.6</version>

		</dependency>

		<dependency>

			<groupId>org.eclipse.persistence</groupId>

			<artifactId>org.eclipse.persistence.moxy</artifactId>

			<version>2.5.0</version>

		</dependency>

		<dependency>

			<groupId>com.sun.jersey.contribs</groupId>

			<artifactId>jersey-spring</artifactId>

			<version>1.8</version>

			<exclusions>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring-core</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring-web</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring-beans</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring-context</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring-utils</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.springframework</groupId>

					<artifactId>spring-aop</artifactId>

				</exclusion>

			</exclusions>

		</dependency>

		<!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->

		<dependency>

			<groupId>com.google.code.gson</groupId>

			<artifactId>gson</artifactId>

			<version>2.3.1</version>

		</dependency>

		<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->

		<dependency>

			<groupId>org.apache.commons</groupId>

			<artifactId>commons-lang3</artifactId>

			<version>3.5</version>

		</dependency>

		<dependency>

			<groupId>io.prometheus</groupId>

			<artifactId>simpleclient</artifactId>

			<version>0.6.0</version>

		</dependency>

		<dependency>

			<groupId>io.prometheus</groupId>

			<artifactId>simpleclient_servlet</artifactId>

			<version>0.6.0</version>

		</dependency>

		<dependency>

			<groupId>io.prometheus</groupId>

			<artifactId>simpleclient_hotspot</artifactId>

			<version>0.6.0</version>

		</dependency>

		<!-- https://mvnrepository.com/artifact/io.micrometer/micrometer-registry-prometheus -->

		<dependency>

			<groupId>io.micrometer</groupId>

			<artifactId>micrometer-registry-prometheus</artifactId>

			<version>1.2.0</version>

		</dependency>

		<!-- https://mvnrepository.com/artifact/io.micrometer/micrometer-spring-legacy -->

		<dependency>

			<groupId>io.micrometer</groupId>

			<artifactId>micrometer-spring-legacy</artifactId>

			<version>1.2.0</version>

		</dependency>

		<!-- https://mvnrepository.com/artifact/org.aspectj/aspectjrt -->

		<!-- AspectJ -->

		<dependency>

			<groupId>org.aspectj</groupId>

			<artifactId>aspectjrt</artifactId>

			<version>1.9.4</version>

		</dependency>

		<dependency>

			<groupId>org.aspectj</groupId>

			<artifactId>aspectjweaver</artifactId>

			<version>1.9.4</version>

		</dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.maven/maven-model -->

        <dependency>

            <groupId>org.apache.maven</groupId>

            <artifactId>maven-model</artifactId>

            <version>3.3.9</version>

        </dependency>

    </dependencies>

	<build>

		<plugins>

			<plugin>

				<groupId>org.apache.maven.plugins</groupId>

				<artifactId>maven-compiler-plugin</artifactId>

				<configuration>

					<source>1.8</source>

					<target>1.8</target>

				</configuration>

			</plugin>

		</plugins>

	</build>

</project>

package eu.dnetlib.data.search.solr;

import eu.dnetlib.api.data.SearchServiceException;

import eu.dnetlib.domain.EPR;

import eu.dnetlib.domain.SecureDriverResource;

import gr.uoa.di.driver.enabling.resultset.ResultSet;

import gr.uoa.di.driver.enabling.resultset.ResultSetFactory;

import io.micrometer.prometheus.PrometheusMeterRegistry;

import org.apache.log4j.Logger;

import org.apache.solr.client.solrj.impl.CloudSolrClient;

import org.springframework.beans.factory.annotation.Autowired;

import org.z3950.zing.cql.CQLParseException;

import java.io.IOException;

import java.util.Arrays;

import java.util.HashMap;

import java.util.Map;

/**

 * Created by antleb on 2/4/14.

 */

public class SolrResultSetFactory implements ResultSetFactory {

    private Logger logger = Logger.getLogger(getClass());

    //private Map<String, CloudSolrClient> clients = new HashMap<String, CloudSolrClient>();

    @Autowired

    private PrometheusMeterRegistry prometheusMeterRegistry;

    private CloudSolrClient solrClient;

    private CloudSolrClient solrClientWithSocketTimeout;

    public void init(String[] zkservers, int socketTimeout) {

        logger.info("Initialization of clients.");

        logger.info("solrClient is set without socket timeout.");

        solrClient = new CloudSolrClient.Builder().withZkHost(Arrays.asList(zkservers)).build();

        if (socketTimeout!=-1) {

            logger.info("solrClientWithSocketTimeout is set with socket timeout " + socketTimeout );

            solrClientWithSocketTimeout = new CloudSolrClient.Builder().withZkHost(Arrays.asList(zkservers))

                    .withSocketTimeout(socketTimeout).build();

        } else {  //sudo solrClientWithSocketTimeout

            logger.info("solrClientWithSocketTimeout has ΝΟ actual timeout as the socketTimeout property is set to -1.");

            solrClientWithSocketTimeout = new CloudSolrClient.Builder().withZkHost(Arrays.asList(zkservers)).build();

        }

    }

    public ResultSet<String> createResultSetWithSocketTimeout(EPR epr) throws IOException, CQLParseException, SearchServiceException {

        logger.info("Get RS with solrClient With SocketTimeout for EPR: " + epr);

        return new SolrResultSet(epr, solrClientWithSocketTimeout, prometheusMeterRegistry);

    }

    public ResultSet<String> createResultSetWithoutSocketTimeout(EPR epr) throws IOException, CQLParseException, SearchServiceException {

        logger.info("Get RS with solrClient Without SocketTimeout for EPR: " + epr);

        return new SolrResultSet(epr, solrClient, prometheusMeterRegistry);

    }

    @Override

    public ResultSet<String> createResultSet(EPR epr) {

        logger.info("Get RS with solrClient Without SocketTimeout for EPR: " + epr);

        try {

            return new SolrResultSet(epr, solrClient, prometheusMeterRegistry);

        } catch (IOException ioe) {

            logger.error("Error in search. I will return null resultSet.", ioe);

        } catch (CQLParseException cqlpe) {

            logger.error("Error in cql parse. I will return null resultSet.", cqlpe);

        } catch (SearchServiceException sse) {

            logger.error("Error in search.", sse);

        }

        return null;

    }

    @Override

    public <D> ResultSet<D> createResultSet(EPR epr, Class<D> resourceClass) {

        return (ResultSet<D>) createResultSet(epr);

    }

    @Override

    public <D> ResultSet<D> createResultSet(ResultSet<?> rs, Class<D> resourceClass) {

        throw new UnsupportedOperationException();

    }

    @Override

    public <D extends SecureDriverResource> ResultSet<D> createSecurityAwareRS(EPR epr, Class<D> resourceClass) {

        throw new UnsupportedOperationException();

    }

}

package eu.dnetlib.data.search.micrometer;

import io.micrometer.core.instrument.Tags;

import io.micrometer.core.instrument.Timer;

import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics;

import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics;

import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics;

import io.micrometer.core.instrument.binder.system.UptimeMetrics;

import io.micrometer.prometheus.PrometheusMeterRegistry;

import org.apache.log4j.Logger;

import org.apache.maven.model.Dependency;

import org.apache.maven.model.Model;

import org.apache.maven.model.io.xpp3.MavenXpp3Reader;

import org.codehaus.plexus.util.xml.pull.XmlPullParserException;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.context.annotation.Configuration;

import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;

import javax.ws.rs.core.Application;

import java.io.IOException;

import java.io.InputStreamReader;

import java.time.Duration;

import java.util.List;

import java.util.concurrent.atomic.AtomicInteger;

@Configuration

@Component

public class Metrics {

    private static final Logger logger = Logger.getLogger(Metrics.class);

    private static final AtomicInteger n = new AtomicInteger(1);

    @Autowired

    private PrometheusMeterRegistry registry;

    @PostConstruct

    public void init() {

        new JvmMemoryMetrics().bindTo(registry);

        new JvmGcMetrics().bindTo(registry);

        new JvmThreadMetrics().bindTo(registry);

        new UptimeMetrics().bindTo(registry);

        Timer.builder("search.server.response.duration").description("search response time with solr request and transformation included")

                .sla(Duration.ofMillis(100), Duration.ofMillis(500), Duration.ofMillis(1000), Duration.ofMillis(2000), Duration.ofMillis(2500),

                        Duration.ofMillis(3000),Duration.ofMillis(4000), Duration.ofMillis(5000)).register(registry);

        Timer.builder("solr.server.response.duration").description("solr response time")

                .sla(Duration.ofMillis(100), Duration.ofMillis(500), Duration.ofMillis(1000), Duration.ofMillis(2000), Duration.ofMillis(2500),

                        Duration.ofMillis(3000),Duration.ofMillis(4000), Duration.ofMillis(5000)).register(registry);

        MavenXpp3Reader reader = new MavenXpp3Reader();

        Model model = null;

        try {

                model = reader.read(

                        new InputStreamReader(

                                Application.class.getResourceAsStream(

                                        "/META-INF/maven/eu.dnetlib/uoa-search/pom.xml"

                                )

                        )

                );

        } catch (XmlPullParserException e) {

           logger.error("Error reading pom. ", e);

        } catch (IOException e) {

            logger.error("Error reading pom. ", e);

        } {

        }

        List<Dependency> list = model.getDependencies();

        Dependency micrometer = list.stream()

                .filter(dependency -> dependency.getArtifactId().equals("micrometer-registry-prometheus"))

                .findFirst()

                .get();

        registry.gauge("micrometer_info", Tags.of("component", model.getArtifactId()).and("scmtag", "HEAD").and("version", model.getVersion()), n);

        //registry.summary("search.server.response.size");

    }

}

package eu.dnetlib.data.search.web.api;

import io.micrometer.prometheus.PrometheusMeterRegistry;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.context.annotation.ComponentScan;

import org.springframework.http.MediaType;

import org.springframework.stereotype.Component;

import org.springframework.stereotype.Controller;

import org.springframework.web.bind.annotation.RequestMapping;

import org.springframework.web.bind.annotation.RequestMethod;

import org.springframework.web.bind.annotation.ResponseBody;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

@Controller

@ComponentScan

@Component

public class MicrometerPublisher {

    @Autowired

    PrometheusMeterRegistry registry;

    @RequestMapping(method = RequestMethod.GET, value = "/metrics", produces = {MediaType.TEXT_PLAIN_VALUE})

    @ResponseBody

    public String getMetrics(HttpServletRequest request, HttpServletResponse response) throws Exception {

        return  registry.scrape();

    }

}

package eu.dnetlib.data.search.web.api;

/**

 * Created by kiatrop on 20/12/2016.

 */

public class TSVResponseFormat {

    public static final String organization = "Name\tProjects\tCountry\n";

    public static final String response = "%s%s";

}

package eu.dnetlib.data.search.web.api;

/**

 * Created by kiatrop on 16/1/2017.

 */

public class HTMLResponseFormat {

    public static final String response = "<table>" +

            "<thead><tr>" +

            " <th>Title</th>" +

            "<th>Authors</th>" +

            "<th>Publication Year</th>" +

            "<th>DOI</th>\n" +

            "<th>Permanent Identifier</th>" +

            "<th>Publication type</th>" +

            "<th>Journal</th>" +

            "<th>Project Name (GA Number)</th>" +

            "<th>Access Mode</th>" +

            "</tr>" +

            "</thead>" +

            "<tbody>%s</tbody></table>";

}

/**

 * 

 */

package eu.dnetlib.data.search.web;

import eu.dnetlib.api.data.SearchService;

import gr.uoa.di.driver.enabling.resultset.ResultSetFactory;

import org.springframework.context.ApplicationContext;

import org.springframework.web.context.support.WebApplicationContextUtils;

import org.w3c.dom.Document;

import org.w3c.dom.Element;

import org.xml.sax.InputSource;

import org.xml.sax.SAXException;

import javax.servlet.ServletContext;

import javax.servlet.http.HttpServletRequest;

import javax.xml.parsers.DocumentBuilder;

import javax.xml.parsers.DocumentBuilderFactory;

import javax.xml.parsers.ParserConfigurationException;

import javax.xml.transform.*;

import javax.xml.transform.dom.DOMSource;

import javax.xml.transform.stream.StreamResult;

import java.io.IOException;

import java.io.StringReader;

import java.io.StringWriter;

/**

 * @author stoumpos

 * 

 */

public class SearchServiceContextUtils {

	public static SearchService getSearchService(HttpServletRequest request) {

		ServletContext session = request.getSession().getServletContext();

		ApplicationContext context = WebApplicationContextUtils

				.getWebApplicationContext(session);

		SearchService search = (SearchService) context.getBean("searchService");

		return search;

	}

	public static ResultSetFactory getResultSetFactory(

			HttpServletRequest request) {

		ServletContext session = request.getSession().getServletContext();

		ApplicationContext context = WebApplicationContextUtils

				.getWebApplicationContext(session);

		ResultSetFactory rsFactory = (ResultSetFactory) context

				.getBean("resultSetFactory");

		return rsFactory;

	}

	public static final String prettyPrint(String xml)

			throws TransformerConfigurationException,

			TransformerFactoryConfigurationError, TransformerException,

			SAXException, IOException, ParserConfigurationException {

		DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();

		dbf.setValidating(true);

		DocumentBuilder db = dbf.newDocumentBuilder();

		InputSource sourceXML = new InputSource(new StringReader(xml));

		Document xmlDoc = db.parse(sourceXML);

		Element source = xmlDoc.getDocumentElement();

		source.normalize();

		StreamResult target = new StreamResult(new StringWriter());

		Transformer tf = TransformerFactory.newInstance().newTransformer();

		tf.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");

		tf.setOutputProperty(OutputKeys.ENCODING, "UTF-8");

		tf.setOutputProperty(OutputKeys.INDENT, "yes");

		tf.transform(new DOMSource(source), target);

		return target.getWriter().toString();

	}

}

package eu.dnetlib.data.search.micrometer;

import io.micrometer.core.aop.TimedAspect;

import io.micrometer.core.instrument.Clock;

import io.micrometer.core.instrument.MeterRegistry;

import io.micrometer.core.instrument.Tag;

import io.micrometer.core.instrument.Tags;

import io.micrometer.prometheus.PrometheusConfig;

import io.micrometer.prometheus.PrometheusMeterRegistry;

import io.micrometer.spring.web.servlet.WebMvcTags;

import io.micrometer.spring.web.servlet.WebMvcTagsProvider;

import io.prometheus.client.CollectorRegistry;

import org.springframework.context.annotation.Bean;

import org.springframework.context.annotation.ComponentScan;

import org.springframework.context.annotation.Configuration;

import org.springframework.stereotype.Component;

import org.springframework.web.servlet.config.annotation.EnableWebMvc;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

import java.util.ArrayList;

@ComponentScan

@Configuration

@Component

@EnableWebMvc

public class MicrometerComponent {

    @Bean

    public Clock micrometerClock() {

        return Clock.SYSTEM;

    }

    @Bean

    public PrometheusConfig prometheusConfig() {

        return PrometheusConfig.DEFAULT;

    }

    @Bean

    public PrometheusMeterRegistry prometheusMeterRegistry(PrometheusConfig config, CollectorRegistry collectorRegistry,

                                                           Clock clock) {

        return new PrometheusMeterRegistry(config, collectorRegistry, clock);

    }

    @Bean

    public CollectorRegistry collectorRegistry() {

        return new CollectorRegistry(true);

    }

    @Bean

    public TimedAspect timedAspect(MeterRegistry registry) {

        return new TimedAspect(registry);

    }

    @Bean

    public WebMvcTagsProvider webMvcTagsProvider() {

        return new WebMvcTagsProvider() {

            @Override

            public Iterable<Tag> httpLongRequestTags(HttpServletRequest httpServletRequest, Object o) {

                return new ArrayList<>();

            }

            @Override

            public Iterable<Tag> httpRequestTags(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, Object o, Throwable throwable) {

                Tag uriTag = Tag.of("status", String.valueOf(httpServletResponse.getStatus()));

                return Tags.of(WebMvcTags.method(httpServletRequest), uriTag, WebMvcTags.exception(throwable), WebMvcTags.status(httpServletResponse));

            }

        };

    }

   /*@Bean

    public MappedInterceptor myInterceptor(PrometheusMeterRegistry prometheusMeterRegistry) {

        return new MappedInterceptor(null, new PrometheusRequestTimerInterceptor(prometheusMeterRegistry));

    }*/

}

package eu.dnetlib.data.search.transform.config;

import eu.dnetlib.data.search.transform.*;

import eu.dnetlib.data.search.transform.formatter.Formatter;

import eu.dnetlib.data.search.transform.formatter.XSLTFormatter;

import eu.dnetlib.data.search.transform.utils.VelocityUtil;

import eu.dnetlib.data.search.utils.vocabulary.VocabularyManager;

import org.apache.commons.io.IOUtils;

import org.apache.log4j.Logger;

import org.apache.velocity.Template;

import org.apache.velocity.VelocityContext;

import org.apache.velocity.app.VelocityEngine;

import java.io.InputStream;

import java.io.StringWriter;

import java.util.*;

public class SearchRegistry implements TransformerRegistry, FormatterRegistry {

	private Logger logger = Logger.getLogger(this.getClass());

	private Configuration config = null;

	private VocabularyManager vocabularyManager = null;

	private Map<Locale,HashMap<String,Transformer>> transformers = new HashMap<Locale, HashMap<String,Transformer>>();

	private Map<String, Formatter> formatters = new HashMap<String, Formatter>();

	@Override

	public Transformer getTransformer(String name, Locale locale)  {

		logger.debug("Requested transformer " + name + " locale " + locale);

		Transformer tx = null;

		if(name == null) {

			return null;

		}

		if (locale == null) {

			locale = config.getDefaultLocale();

		}

		try {

			if (this.transformers.get(locale)!= null) {

				tx = this.transformers.get(locale).get(name);

			} else { //not map for locale

				logger.debug("Transformation map " + config.getTransformationsMap().keySet());

				tx = createChainTransformer(name, locale,  config.getTransformationsMap().get(name));

				Map<String, Transformer> map = new HashMap<String, Transformer>();

				map.put(name, tx);

				transformers.put(locale, (HashMap<String, Transformer>) map);

			}

			if (tx == null) { // there is the local map but not the transformer

				tx = createChainTransformer(name, locale,  config.getTransformationsMap().get(name));

				transformers.get(locale).put(name, tx);

			}

		} catch (Exception e) {

			logger.error("Cannot create transformer " + name, e);

			throw new IllegalArgumentException("Cannot create transformer " + name, e);

		}

		logger.debug("Returned in get transformer" + tx);

		return tx;

	}

	@Override

	public Formatter getFormatter(String name) {

		Formatter fm = this.formatters.get(name);

		try {

			if (fm == null) {

				FormatterConfiguration formatterConfiguration = config.getFormattersMap().get(name);

				String formatterXslt = formatterConfiguration.getXsl_file();

				String template = formatterConfiguration.getTemplate();

				logger.debug("Configuration for formatter: " + formatterXslt + ", " + template);

				if (formatterXslt == null) {

					logger.error("Cannot create formatter " + name);

					throw new IllegalArgumentException("Cannot create formatter " + name);

				}

				if (template == null) {

					fm = createFormatter(formatterXslt);

				} else {

					fm = createFormatter(formatterXslt, template);

				}

				formatters.put(name, fm);

			}

		} catch (Exception e) {

			logger.error("Cannot create formatter " + name);

			throw new IllegalArgumentException("Cannot create formatter " + name, e);

		}

		return fm;

	}

	private Transformer createChainTransformer(String name, Locale locale, List<Transformation> transformations) throws SearchFactoryException {

		List<XPathTrasformation> temp = new ArrayList<XPathTrasformation>();

		List<Transformer> forChaining = new ArrayList<Transformer>();

		for (Transformation tr: transformations){

			if (tr instanceof XSLTTransformation) {

				try {

					if (!temp.isEmpty()){

						forChaining.add(createTransformer(locale, temp));

					}

					temp.clear();

					forChaining.add(createTransformer((XSLTTransformation)tr));

				} catch (Exception e) {

					logger.warn("Error creating a transformer with name: "+name+" and locale: "+locale.getDisplayName(), e);

					throw new SearchFactoryException("Error creating a transformer with name: "+name+" and locale: "+locale.getDisplayName(), e);

				}

			}

			else {

				temp.add((XPathTrasformation)tr);

			}

		}

		if (!temp.isEmpty()){

			try {

				forChaining.add(createTransformer(locale, temp));

			} catch (Exception e) {

				logger.warn("Error creating a transformer with name: " + name + " and locale: "+locale.getDisplayName(), e);

				throw new SearchFactoryException("Error creating a transformer with name: " + name + " and locale: "+locale.getDisplayName(), e);

			}

		}

		return new ChainTransformer(name, forChaining);

	}

	private Transformer createTransformer(XSLTTransformation transformation) throws Exception {

		String xsl_file = transformation.getXslt();

		InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(xsl_file);

		StringWriter writer = new StringWriter();

		IOUtils.copy(inputStream, writer);

		String xsl = writer.toString();

		XsltTransformer OptimusPrime = new XsltTransformer(xsl);

		writer.close();

		IOUtils.closeQuietly(inputStream);

		return OptimusPrime;

	}

	private Transformer createTransformer(Locale locale, List<XPathTrasformation> transformations) throws Exception {

		VelocityEngine ve = VelocityUtil.getEngine();

		Template t = ve.getTemplate("/eu/dnetlib/data/search/transform/config/velocity.vm", "UTF-8");

		VelocityContext context = new VelocityContext();

		context.put("locale", locale);

		context.put("defLocale", config.getDefaultLocale());

		context.put("transformations", transformations);

		StringWriter writer = new StringWriter();

		t.merge(context, writer);

		XsltTransformer Megatron = new XsltTransformer(writer.toString());

		//logger.debug("Transformer xsl " + Megatron.getXslt());

		//logger.debug("Vocab paths are: " + loadVocabulariesPathMap(transformations, locale).toString());

		return Megatron;

	}

	private Formatter createFormatter(String xsl_file) throws Exception {

		logger.debug("Loading xslt " + xsl_file);

		InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(xsl_file);

		StringWriter writer = new StringWriter();

		IOUtils.copy(inputStream, writer);

		String xsl = writer.toString();

		XSLTFormatter Cybertron = new XSLTFormatter(xsl);

		writer.close();

		IOUtils.closeQuietly(inputStream);

		return Cybertron;

	}

	private Formatter createFormatter(String xsl_file, String template) throws Exception {

		logger.debug("Loading xslt " + xsl_file);

		logger.debug("Loading template " + template);

		InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(xsl_file);

		StringWriter writer = new StringWriter();

		IOUtils.copy(inputStream, writer);

		String xsl = writer.toString();

		XSLTFormatter Cybertron = new XSLTFormatter(xsl, template);

		writer.close();

		IOUtils.closeQuietly(inputStream);

		return Cybertron;

	}

	public Configuration getConfig() {

		return config;

	}

	public void setConfig(Configuration config) {

		this.config = config;

	}

	public void setVocabularyManager(VocabularyManager vocabularyManager) {

		this.vocabularyManager = vocabularyManager;

	}

	public VocabularyManager getVocabularyManager() {

		return vocabularyManager;

	}

}

package eu.dnetlib.data.search.app.plan;

/**

 */

public abstract class FieldRewriteRule {

	protected String name = null;

	protected String fieldName = null;

	public FieldRewriteRule(String name, String fieldName) {

		this.name = name;

		this.fieldName = fieldName;

	}

	public String getFieldName() {

		return fieldName;

	}

	public void setFieldName(String fieldName) {

		this.fieldName = fieldName;

	}

	public String getName() {

		return name;

	}

	public void setName(String name) {

		this.name = name;

	}

	//public abstract CqlClause apply(CqlRelation relation) throws CqlException;

	@Override

	public String toString() {

		return name;

	}

}

package eu.dnetlib.data.search.web.api;

import io.micrometer.prometheus.PrometheusMeterRegistry;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.http.HttpStatus;

import org.springframework.http.ResponseEntity;

import org.springframework.web.bind.annotation.ControllerAdvice;

import org.springframework.web.bind.annotation.ExceptionHandler;

import org.springframework.web.servlet.config.annotation.EnableWebMvc;

import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler;

@EnableWebMvc

@ControllerAdvice(value = "eu.dnetlib.data.search.web.api", assignableTypes = SearchRequestController.class)

public class ExceptionController extends ResponseEntityExceptionHandler{

    @Autowired

    PrometheusMeterRegistry registry;

    @ExceptionHandler(Exception.class)

    public ResponseEntity<Error> invalidInput(Exception ex) {

        Error response = new Error();

        response.setStatus("error");

        response.setCode("500");

        response.setMessage(ex.getMessage());

        response.setException(ex.getCause().getMessage());

        registry.counter("http.status.500", "500", "uri").increment();

        return new ResponseEntity<Error>(response, HttpStatus.CONFLICT);

    }

}

package eu.dnetlib.data.search.web.api;

import eu.dnetlib.data.search.web.utils.RequestResponseHandler;

import org.apache.log4j.Logger;

/**

 * Created by kiatrop on 20/12/2016.

 */

public class CSVResponseFormat {

    public static final String organization = "Name,Projects,Country\r\n";

    public static final String datasource = "Name,Type,Compatibility,OAI-PMH,website\r\n";

    public static final String service = "Name,website\r\n";

    public static final String person = "Fullname\r\n";

    public static final String project = "Title,Funder,Start Date,End Date,Participants\r\n";

    //    public static final String result = "Type,Title,Authors,Publication Year,DOI,Green,OA colour,Download from,Type,Diamond,Journal,Publicly Funded,Funder|Project Name (GA Number),Access\r\n";

    public static final String result = "Type,Title,Authors,Publication Year,DOI,Type,Peer Reviewed,Diamond Journal,Journal,Download from,Publicly Funded,Funder|Project Name (GA Number),Access,Green,OA colour\r\n";

    //TODO SPECIAL

    public static final String result_special = "Type,Title,Authors,Publication Year,DOI,Type,Peer Reviewed,Diamond Journal,Journal,Download from,Publicly Funded,Funder,Project Name (GA Number),Access,Green,OA colour\r\n";

    private static Logger logger = Logger.getLogger(CSVResponseFormat.class);

    public static final String response = "%s%s";

    public static String appendTitle(RequestResponseHandler.Entity entity, boolean special) {

        if(entity.equals(RequestResponseHandler.Entity.ORGANIZATION)) {

            return  organization;

        } else if (entity.equals(RequestResponseHandler.Entity.DATASOURCE)) {

            return  datasource;

        } else if (entity.equals(RequestResponseHandler.Entity.SERVICE)) {

            return service;

        }

        /* else if (entity.equals(RequestResponseHandler.Entity.PERSON)) {

            return  person;

        }*/ else if (entity.equals(RequestResponseHandler.Entity.PROJECT)) {

            return project;

        } else if (entity.equals(RequestResponseHandler.Entity.PUBLICATION) || entity.equals(RequestResponseHandler.Entity.DATASET)

                    ||entity.equals(RequestResponseHandler.Entity.SOFTWARE) || entity.equals(RequestResponseHandler.Entity.OTHER)

                    || entity.equals(RequestResponseHandler.Entity.RESULT)) {

            if (special) {

                return result_special;

            }

            return result;

        }

        return "";

    }

}

package eu.dnetlib.data.search.transform.config.vocabularies;

/**

 * Created by kiatrop on 18/6/2014.

 */

public class FormatterParams {

    String xsl_file;

    String template_file;

    public String getXsl_file() {

        return xsl_file;

    }

    public void setXsl_file(String xsl_file) {

        this.xsl_file = xsl_file;

    }

    public String getTemplate_file() {

        return template_file;

    }

    public void setTemplate_file(String template_file) {

        this.template_file = template_file;

    }

}

package eu.dnetlib.data.search.transform.config;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.List;

public class XPathTrasformation extends Transformation {

	private String change;

	private String vocabulary;

	private List<String> matches;

	public XPathTrasformation(String match, String change, String vocabulary) {

		this.change = change;

		this.vocabulary = vocabulary;

		this.matches = new ArrayList<String>(Arrays.asList(match.split("\\.")));

	}

	public List<String> getMatches() {

		return matches;

	}

	public String getChange() {

		return change;

	}

	public void setChange(String change) {

		this.change = change;

	}

	public String getVocabulary() {

		return vocabulary;

	}

	public void setVocabulary(String vocabulary) {

		this.vocabulary = vocabulary;

	}

}

package eu.dnetlib.data.search.solr;

import com.google.gson.Gson;

import eu.dnetlib.api.data.SearchServiceException;

import eu.dnetlib.data.search.transform.Transformer;

import eu.dnetlib.data.search.transform.TransformerException;

import eu.dnetlib.data.search.utils.solr.SolrResultSetOptionsUtil;

import eu.dnetlib.data.search.utils.solr.SolrResultsFormatter;

import eu.dnetlib.domain.EPR;

import gr.uoa.di.driver.enabling.resultset.ResultSet;

import io.micrometer.core.instrument.Timer;

import io.micrometer.prometheus.PrometheusMeterRegistry;

import org.apache.commons.lang.StringEscapeUtils;

import org.apache.log4j.Logger;

import org.apache.solr.client.solrj.SolrClient;

import org.apache.solr.client.solrj.SolrServerException;

import org.apache.solr.client.solrj.impl.CloudSolrClient;

import org.apache.solr.client.solrj.response.FacetField;

import org.apache.solr.client.solrj.response.QueryResponse;

import org.apache.solr.common.SolrDocumentList;

import org.apache.solr.common.params.SolrParams;

import org.apache.solr.common.util.NamedList;

import org.z3950.zing.cql.CQLParseException;

import javax.ws.rs.core.MediaType;

import java.io.IOException;

import java.io.OutputStream;

import java.util.*;

/**

 * Created by antleb on 2/4/14.

 */

public class SolrResultSet implements ResultSet<String> {

    private Logger logger = Logger.getLogger(getClass());

    private EPR epr = null;

    public SolrClient solrClient = null;

    private NamedList<String> queryOpts = new NamedList<String>();

    long size = -1;

    private PrometheusMeterRegistry registry;

    public SolrResultSet(EPR epr, CloudSolrClient cloudSolrClient, PrometheusMeterRegistry registry)

            throws IOException, CQLParseException, SearchServiceException {

        this.epr = epr;

        this.solrClient = cloudSolrClient;

        this.queryOpts = SolrResultSetOptionsUtil.extractQueryOptions(epr.getParameter("query"));

        String layout = epr.getParameter("layout");

        String mdformat = epr.getParameter("mdformat");

        String interpretation = epr.getParameter("interpretation");

        ((CloudSolrClient)solrClient).setDefaultCollection(mdformat + "-" + layout + "-" + interpretation);

        this.registry = registry;

    }

    @Override

    public boolean isOpen() {

        return true;

    }

    @Override

    public boolean isAlive() {

        return true;

    }

    @Override

    public void close() {

        /* Closes the whole pull. We do not want this.

        try {

            logger.info("!!!!!!!!! !!!!!!! CLOSING !!!!!!!!! !!!!!!!!!! ");

            solrClient.close();

        } catch (IOException e) {

            logger.error("Error closing result set.", e);

        }*/

    }

    @Override

    public int size() {

        return (int) size;

    }

    @Override

    @Deprecated

    public List<String> getElements(int from, int to) {

        return get(from, to);

    }

    List<FacetField> facetFields = null;

    @Override

    @Deprecated

    public List<String> get(int from, int to) {

        List<String> res = new ArrayList<String>();

        QueryResponse rsp = null;

        HashMap<String, List<String>> map = new HashMap<String, List<String>>();

        logger.debug("from: " + from);

        logger.debug("to: " + to);

        queryOpts.add("start", from+1 + "");

        queryOpts.add("rows", to + 1+"");

        try {

            rsp = solrClient.query(SolrParams.toSolrParams(queryOpts));

            facetFields = rsp.getFacetFields();

            SolrDocumentList docs = rsp.getResults();

            if (facetFields!=null && !facetFields.isEmpty()) {

                for (int i = from - 1; i < to; i++) {

                    for (FacetField field : facetFields) {

                        if (field.getValueCount() > i) {

                            BrowseField bf = new BrowseField();

                            bf.setId(field.getValues().get(i).getName());

                            bf.setName(field.getValues().get(i).getName());

                            bf.setCount(field.getValues().get(i).getCount() + "");

                            if (map.get(field.getName()) == null) {

                                map.put(field.getName(), new ArrayList<String>());

                            }

                            map.get(field.getName()).add(new Gson().toJson(bf));

                        }

                    }

                }

                for (Map.Entry<String, List<String>> facetEntry : map.entrySet()) {

                    StringBuilder builder = new StringBuilder();

                    builder.append("\"" + facetEntry.getKey() + "\"" + " : ");

                    builder.append(facetEntry.getValue());

                    res.add(builder.toString());

                }

            }

            logger.debug("time: " + rsp.getElapsedTime());

            logger.debug("found: " + docs.getNumFound());

            logger.debug("docs: " + docs.size());

            for (int i = 0; i < docs.size(); i++) {

                String result = (String) docs.get(i).get("__result");

                res.add(result);

            }

            return res;

        } catch (SolrServerException sse) {

            logger.error("Fail to get results from Solr. ", sse);

        } catch (IOException ioe) {

            logger.error("Fail to get results from Solr. ", ioe);

        }

        return null;

    }

    @Override

    public EPR getEpr() {

        return null;

    }

    public Map<String,List<String>> newGet(int from, int to, String format, Transformer transformer,

                                           Transformer oldRefineTransformer, boolean timeout, String timeoutValue) throws SearchServiceException {

        List<String> refineSolrResults = new ArrayList<String>();

        List<String> searchSolrResults = new ArrayList<String>();

        QueryResponse rsp = null;

        HashMap<String, List<String>> map = new HashMap<String, List<String>>();

        if (from < 0) from = 0;

        if (to < 0) to = 0;

        queryOpts.add("start", from*to + "");

        queryOpts.add("rows", to +"");

        try {

            if (timeout) {

                if (timeoutValue != null && Integer.valueOf(timeoutValue) == -1) {

                    logger.warn("Timeout value is -1. Timeout will not be set for query: " + queryOpts.get("q"));

                    timeout = false;

                } else if (Integer.valueOf(timeoutValue) < 500) {

                    logger.warn("Timeout is less than 500ms. We fall back to the default 1000ms for query:" + queryOpts.get("q"));

                    queryOpts.add("timeAllowed", "500");

                } else if (timeout) {

                    queryOpts.add("timeAllowed", timeoutValue);

                }

            }

        } catch (NumberFormatException nfe) {

            logger.warn("NumberFormatException occurred. Timeout not properly set. We fall back to the default 1000ms.");

            queryOpts.add("timeAllowed", "500");

        }

        logger.info("timeout: " + timeout + ", timeout value: " + queryOpts.get("timeAllowed"));

        //queryOpts.add("f.resulthostingdatasource.facet.limit", "2");

        long startTime = System.nanoTime();

        try {

            io.micrometer.core.instrument.Timer.Sample timer = Timer.start(registry);

            rsp = solrClient.query(SolrParams.toSolrParams(queryOpts));

            timer.stop(registry.timer("solr.server.response.duration"));

            long estimatedTime = System.nanoTime() - startTime;

            logger.info("Solrj time " + estimatedTime/1000000 +  " milliseconds for query:" + queryOpts.get("q") +

                    " and facets " + queryOpts.getAll("facet.field") + " and fq " + queryOpts.getAll("fq") +

                    " and sorts " + queryOpts.getAll("sort") + " from: "

                    + from + " and size " + to + " with timeout: " + timeout);

            /*System.out.println("Solrj time " + estimatedTime/1000000 +  " milliseconds for query:" + queryOpts.get("q") +

                    " and facets " + queryOpts.getAll("facet.field") + " and fq " + queryOpts.getAll("fq") +

                    " and sorts " + queryOpts.getAll("sort") + " from: "

                    + from + " and size " + to);

            */

            facetFields = rsp.getFacetFields();

            SolrDocumentList docs = rsp.getResults();

            this.size = docs.getNumFound();

            if (facetFields!=null && !facetFields.isEmpty()) {

                if (format != null && format.equals(MediaType.APPLICATION_JSON)) {

                    for (FacetField field : facetFields) {

                        map.put(field.getName(), new ArrayList<String>());

                        BrowseField bf = null;

                        for (int i = 0; i < field.getValueCount(); i++) {

                            bf = new BrowseField();

                            //bf.setId(org.apache.commons.lang3.StringEscapeUtils.escapeJson(field.getValues().get(i).getName()));

                            bf.setId(field.getValues().get(i).getName());

                            String[] facetedValues = field.getValues().get(i).getName().split("\\|\\|",2);

                            logger.debug("faceted values " + Arrays.toString(facetedValues));

                            if (facetedValues.length > 1) {

                                //bf.setName(org.apache.commons.lang3.StringEscapeUtils.escapeJson(facetedValues[1]));

                                bf.setName(facetedValues[1]);

                                logger.debug("faceted values [1] " + facetedValues[1]);

                            } else if (field.getValues().get(i).getName().split("_\\:\\:",2).length > 1) {

                                //bf.setName(org.apache.commons.lang3.StringEscapeUtils.escapeJson(field.getValues().get(i).getName().split("\\:\\:",2)[1]).replaceAll("\\:\\:", "\\|"));

                                bf.setName(field.getValues().get(i).getName().split("\\:\\:",2)[1].replaceAll("\\:\\:", "\\|"));

                            } else {

                                //bf.setName(org.apache.commons.lang3.StringEscapeUtils.escapeJson(field.getValues().get(i).getName()));

                                bf.setName(field.getValues().get(i).getName());

                            }

                            bf.setCount(field.getValues().get(i).getCount() + "");

                            map.get(field.getName()).add(new Gson().toJson(bf));

                        }

                    }

                    StringBuilder builder = null;

                    for (Map.Entry<String, List<String>> facetEntry : map.entrySet()) {

                        builder = new StringBuilder();

                        builder.append("\"" + facetEntry.getKey() + "\"" + " : ");

                        builder.append(facetEntry.getValue());

                        refineSolrResults.add(builder.toString());

                    }