Revision 63330
Added by Katerina Iatropoulou 12 days ago
modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/app/plan/QueryRewriteRule.java | ||
---|---|---|
1 |
package eu.dnetlib.data.search.app.plan; |
|
2 |
|
|
3 |
/** |
|
4 |
*/ |
|
5 |
public abstract class QueryRewriteRule { |
|
6 |
|
|
7 |
protected String name = null; |
|
8 |
|
|
9 |
public QueryRewriteRule(String name) { |
|
10 |
this.name = name; |
|
11 |
} |
|
12 |
|
|
13 |
public QueryRewriteRule() { |
|
14 |
this(""); |
|
15 |
} |
|
16 |
|
|
17 |
public String getName() { |
|
18 |
return name; |
|
19 |
} |
|
20 |
|
|
21 |
public void setName(String name) { |
|
22 |
this.name = name; |
|
23 |
} |
|
24 |
|
|
25 |
public abstract String apply(String query); |
|
26 |
|
|
27 |
@Override |
|
28 |
public String toString() { |
|
29 |
return "'"+name+"'"; |
|
30 |
} |
|
31 |
} |
|
0 | 32 |
modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/app/plan/CollectionRewriteRule.java | ||
---|---|---|
1 |
package eu.dnetlib.data.search.app.plan; |
|
2 |
|
|
3 |
import eu.dnetlib.domain.functionality.Collection; |
|
4 |
import gr.uoa.di.driver.enabling.ISLookUp; |
|
5 |
import org.apache.log4j.Logger; |
|
6 |
|
|
7 |
public class CollectionRewriteRule extends FieldRewriteRule { |
|
8 |
|
|
9 |
public static Logger logger = Logger.getLogger(QueryRewriteRule.class); |
|
10 |
|
|
11 |
private ISLookUp<Collection> collectionLookUp = null; |
|
12 |
|
|
13 |
public CollectionRewriteRule() { |
|
14 |
super(null, null); |
|
15 |
} |
|
16 |
|
|
17 |
public CollectionRewriteRule(String name, String fieldName) { |
|
18 |
super(name, fieldName); |
|
19 |
} |
|
20 |
|
|
21 |
public ISLookUp<Collection> getCollectionLookUp() { |
|
22 |
return collectionLookUp; |
|
23 |
} |
|
24 |
|
|
25 |
public void setCollectionLookUp( |
|
26 |
ISLookUp<eu.dnetlib.domain.functionality.Collection> collectionLookUp) { |
|
27 |
this.collectionLookUp = collectionLookUp; |
|
28 |
} |
|
29 |
|
|
30 |
/* @Override |
|
31 |
public CqlClause apply(CqlRelation relation) throws CqlException { |
|
32 |
|
|
33 |
if (!relation.getIndex().equals(fieldName)) { |
|
34 |
throw new IllegalArgumentException("Rule " + name |
|
35 |
+ " expects field " + fieldName |
|
36 |
+ " but is applied on field " + relation.getIndex()); |
|
37 |
} |
|
38 |
|
|
39 |
String id = relation.getValue().substring(1, relation.getValue().length()-1); |
|
40 |
try { |
|
41 |
Collection collection = getCollectionLookUp().getById(id); |
|
42 |
if (logger.isDebugEnabled()) { |
|
43 |
logger.debug("rewrite relation '" + relation.toCqlString() |
|
44 |
+ "' as '" + collection.getRetrievalCondition() + "'"); |
|
45 |
} |
|
46 |
|
|
47 |
Query query = new Query(collection.getRetrievalCondition()); |
|
48 |
return query.getCqlQuery().getRoot(); |
|
49 |
|
|
50 |
} catch (ISLookUpException e) { |
|
51 |
throw new CqlException( |
|
52 |
"Collection lookup for " + id + " failed.", e); |
|
53 |
} |
|
54 |
} */ |
|
55 |
} |
modules/uoa-search/branches/uoa-search-json/pom.xml | ||
---|---|---|
1 |
<?xml version="1.0"?> |
|
2 |
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> |
|
3 |
<parent> |
|
4 |
<groupId>eu.dnetlib</groupId> |
|
5 |
<artifactId>dnet45-parent</artifactId> |
|
6 |
<version>1.0.0</version> |
|
7 |
</parent> |
|
8 |
<modelVersion>4.0.0</modelVersion> |
|
9 |
<groupId>eu.dnetlib</groupId> |
|
10 |
<artifactId>uoa-search</artifactId> |
|
11 |
<packaging>jar</packaging> |
|
12 |
<version>3.13.5-SNAPSHOT</version> |
|
13 |
<scm> |
|
14 |
<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/uoa-search/branches/uoa-search-pre-3.9.4</developerConnection> |
|
15 |
</scm> |
|
16 |
|
|
17 |
<repositories> |
|
18 |
<!-- Cloudera Repositories --> |
|
19 |
<repository> |
|
20 |
<snapshots> |
|
21 |
<enabled>false</enabled> |
|
22 |
</snapshots> |
|
23 |
<id>cloudera-central</id> |
|
24 |
<name>cloundera-libs-release</name> |
|
25 |
<url>http://maven.research-infrastructures.eu/nexus/content/repositories/cloudera-central</url> |
|
26 |
</repository> |
|
27 |
<repository> |
|
28 |
<id>cloudera-snapshots</id> |
|
29 |
<name>cloudera-libs-snapshot</name> |
|
30 |
<url>http://maven.research-infrastructures.eu/nexus/content/repositories/cloudera-snapshots</url> |
|
31 |
</repository> |
|
32 |
<repository> |
|
33 |
<id>typesafe</id> |
|
34 |
<name>typesafe-releases</name> |
|
35 |
<url>http://maven.research-infrastructures.eu/nexus/content/repositories/typesafe</url> |
|
36 |
</repository> |
|
37 |
</repositories> |
|
38 |
|
|
39 |
<dependencies> |
|
40 |
<dependency> |
|
41 |
<groupId>junit</groupId> |
|
42 |
<artifactId>junit</artifactId> |
|
43 |
<version>${junit.version}</version> |
|
44 |
<scope>test</scope> |
|
45 |
</dependency> |
|
46 |
<dependency> |
|
47 |
<groupId>eu.dnetlib</groupId> |
|
48 |
<artifactId>uoa-commons</artifactId> |
|
49 |
<version>[2.0.0, 3.0.0)</version> |
|
50 |
</dependency> |
|
51 |
<dependency> |
|
52 |
<groupId>org.mockito</groupId> |
|
53 |
<artifactId>mockito-all</artifactId> |
|
54 |
<version>1.6</version> |
|
55 |
</dependency> |
|
56 |
<dependency> |
|
57 |
<groupId>org.apache.solr</groupId> |
|
58 |
<artifactId>solr-solrj</artifactId> |
|
59 |
<version>7.2.1</version> |
|
60 |
<exclusions> |
|
61 |
<exclusion> |
|
62 |
<groupId>org.apache.zookeeper</groupId> |
|
63 |
<artifactId>zookeeper</artifactId> |
|
64 |
</exclusion> |
|
65 |
</exclusions> |
|
66 |
</dependency> |
|
67 |
<dependency> |
|
68 |
<groupId>org.apache.zookeeper</groupId> |
|
69 |
<artifactId>zookeeper</artifactId> |
|
70 |
<version>3.4.5-cdh4.3.0</version> |
|
71 |
</dependency> |
|
72 |
<dependency> |
|
73 |
<groupId>eu.dnetlib</groupId> |
|
74 |
<artifactId>cnr-cql-utils</artifactId> |
|
75 |
<version>[2.0.0, 3.0.0)</version> |
|
76 |
<scope>compile</scope> |
|
77 |
</dependency> |
|
78 |
<dependency> |
|
79 |
<groupId>eu.dnetlib</groupId> |
|
80 |
<artifactId>cnr-blackboard-common</artifactId> |
|
81 |
<version>[2.2.1, 3.0.0)</version> |
|
82 |
</dependency> |
|
83 |
|
|
84 |
<!-- Spring MVC framework --> |
|
85 |
<dependency> |
|
86 |
<groupId>org.springframework</groupId> |
|
87 |
<artifactId>spring-webmvc</artifactId> |
|
88 |
<version>${spring.version}</version> |
|
89 |
</dependency> |
|
90 |
|
|
91 |
<dependency> |
|
92 |
<groupId>org.apache.velocity</groupId> |
|
93 |
<artifactId>velocity</artifactId> |
|
94 |
<version>1.7</version> |
|
95 |
</dependency> |
|
96 |
|
|
97 |
<dependency> |
|
98 |
<groupId>asm</groupId> |
|
99 |
<artifactId>asm</artifactId> |
|
100 |
<version>3.3.1</version> |
|
101 |
</dependency> |
|
102 |
<dependency> |
|
103 |
<groupId>com.sun.jersey</groupId> |
|
104 |
<artifactId>jersey-bundle</artifactId> |
|
105 |
<version>1.19</version> |
|
106 |
</dependency> |
|
107 |
<dependency> |
|
108 |
<groupId>org.json</groupId> |
|
109 |
<artifactId>json</artifactId> |
|
110 |
<version>20140107</version> |
|
111 |
</dependency> |
|
112 |
<dependency> |
|
113 |
<groupId>com.sun.jersey</groupId> |
|
114 |
<artifactId>jersey-server</artifactId> |
|
115 |
<version>1.19</version> |
|
116 |
</dependency> |
|
117 |
<dependency> |
|
118 |
<groupId>com.sun.jersey</groupId> |
|
119 |
<artifactId>jersey-core</artifactId> |
|
120 |
<version>1.19</version> |
|
121 |
</dependency> |
|
122 |
<dependency> |
|
123 |
<groupId>commons-validator</groupId> |
|
124 |
<artifactId>commons-validator</artifactId> |
|
125 |
<version>1.6</version> |
|
126 |
</dependency> |
|
127 |
<dependency> |
|
128 |
<groupId>org.eclipse.persistence</groupId> |
|
129 |
<artifactId>org.eclipse.persistence.moxy</artifactId> |
|
130 |
<version>2.5.0</version> |
|
131 |
</dependency> |
|
132 |
<dependency> |
|
133 |
<groupId>com.sun.jersey.contribs</groupId> |
|
134 |
<artifactId>jersey-spring</artifactId> |
|
135 |
<version>1.8</version> |
|
136 |
<exclusions> |
|
137 |
<exclusion> |
|
138 |
<groupId>org.springframework</groupId> |
|
139 |
<artifactId>spring</artifactId> |
|
140 |
</exclusion> |
|
141 |
<exclusion> |
|
142 |
<groupId>org.springframework</groupId> |
|
143 |
<artifactId>spring-core</artifactId> |
|
144 |
</exclusion> |
|
145 |
<exclusion> |
|
146 |
<groupId>org.springframework</groupId> |
|
147 |
<artifactId>spring-web</artifactId> |
|
148 |
</exclusion> |
|
149 |
<exclusion> |
|
150 |
<groupId>org.springframework</groupId> |
|
151 |
<artifactId>spring-beans</artifactId> |
|
152 |
</exclusion> |
|
153 |
<exclusion> |
|
154 |
<groupId>org.springframework</groupId> |
|
155 |
<artifactId>spring-context</artifactId> |
|
156 |
</exclusion> |
|
157 |
<exclusion> |
|
158 |
<groupId>org.springframework</groupId> |
|
159 |
<artifactId>spring-utils</artifactId> |
|
160 |
</exclusion> |
|
161 |
<exclusion> |
|
162 |
<groupId>org.springframework</groupId> |
|
163 |
<artifactId>spring-aop</artifactId> |
|
164 |
</exclusion> |
|
165 |
</exclusions> |
|
166 |
</dependency> |
|
167 |
<!-- https://mvnrepository.com/artifact/com.google.code.gson/gson --> |
|
168 |
<dependency> |
|
169 |
<groupId>com.google.code.gson</groupId> |
|
170 |
<artifactId>gson</artifactId> |
|
171 |
<version>2.3.1</version> |
|
172 |
</dependency> |
|
173 |
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 --> |
|
174 |
<dependency> |
|
175 |
<groupId>org.apache.commons</groupId> |
|
176 |
<artifactId>commons-lang3</artifactId> |
|
177 |
<version>3.5</version> |
|
178 |
</dependency> |
|
179 |
|
|
180 |
<dependency> |
|
181 |
<groupId>io.prometheus</groupId> |
|
182 |
<artifactId>simpleclient</artifactId> |
|
183 |
<version>0.6.0</version> |
|
184 |
</dependency> |
|
185 |
|
|
186 |
<dependency> |
|
187 |
<groupId>io.prometheus</groupId> |
|
188 |
<artifactId>simpleclient_servlet</artifactId> |
|
189 |
<version>0.6.0</version> |
|
190 |
</dependency> |
|
191 |
|
|
192 |
<dependency> |
|
193 |
<groupId>io.prometheus</groupId> |
|
194 |
<artifactId>simpleclient_hotspot</artifactId> |
|
195 |
<version>0.6.0</version> |
|
196 |
</dependency> |
|
197 |
|
|
198 |
<!-- https://mvnrepository.com/artifact/io.micrometer/micrometer-registry-prometheus --> |
|
199 |
<dependency> |
|
200 |
<groupId>io.micrometer</groupId> |
|
201 |
<artifactId>micrometer-registry-prometheus</artifactId> |
|
202 |
<version>1.2.0</version> |
|
203 |
</dependency> |
|
204 |
|
|
205 |
<!-- https://mvnrepository.com/artifact/io.micrometer/micrometer-spring-legacy --> |
|
206 |
<dependency> |
|
207 |
<groupId>io.micrometer</groupId> |
|
208 |
<artifactId>micrometer-spring-legacy</artifactId> |
|
209 |
<version>1.2.0</version> |
|
210 |
</dependency> |
|
211 |
|
|
212 |
|
|
213 |
<!-- https://mvnrepository.com/artifact/org.aspectj/aspectjrt --> |
|
214 |
<!-- AspectJ --> |
|
215 |
<dependency> |
|
216 |
<groupId>org.aspectj</groupId> |
|
217 |
<artifactId>aspectjrt</artifactId> |
|
218 |
<version>1.9.4</version> |
|
219 |
</dependency> |
|
220 |
<dependency> |
|
221 |
<groupId>org.aspectj</groupId> |
|
222 |
<artifactId>aspectjweaver</artifactId> |
|
223 |
<version>1.9.4</version> |
|
224 |
</dependency> |
|
225 |
|
|
226 |
<!-- https://mvnrepository.com/artifact/org.apache.maven/maven-model --> |
|
227 |
<dependency> |
|
228 |
<groupId>org.apache.maven</groupId> |
|
229 |
<artifactId>maven-model</artifactId> |
|
230 |
<version>3.3.9</version> |
|
231 |
</dependency> |
|
232 |
|
|
233 |
|
|
234 |
|
|
235 |
</dependencies> |
|
236 |
|
|
237 |
<build> |
|
238 |
<plugins> |
|
239 |
<plugin> |
|
240 |
<groupId>org.apache.maven.plugins</groupId> |
|
241 |
<artifactId>maven-compiler-plugin</artifactId> |
|
242 |
<configuration> |
|
243 |
<source>1.8</source> |
|
244 |
<target>1.8</target> |
|
245 |
</configuration> |
|
246 |
</plugin> |
|
247 |
</plugins> |
|
248 |
</build> |
|
249 |
</project> |
modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/solr/SolrResultSetFactory.java | ||
---|---|---|
1 |
package eu.dnetlib.data.search.solr; |
|
2 |
|
|
3 |
import eu.dnetlib.api.data.SearchServiceException; |
|
4 |
import eu.dnetlib.domain.EPR; |
|
5 |
import eu.dnetlib.domain.SecureDriverResource; |
|
6 |
import gr.uoa.di.driver.enabling.resultset.ResultSet; |
|
7 |
import gr.uoa.di.driver.enabling.resultset.ResultSetFactory; |
|
8 |
import io.micrometer.prometheus.PrometheusMeterRegistry; |
|
9 |
import org.apache.log4j.Logger; |
|
10 |
import org.apache.solr.client.solrj.impl.CloudSolrClient; |
|
11 |
import org.springframework.beans.factory.annotation.Autowired; |
|
12 |
import org.z3950.zing.cql.CQLParseException; |
|
13 |
|
|
14 |
import java.io.IOException; |
|
15 |
import java.util.Arrays; |
|
16 |
import java.util.HashMap; |
|
17 |
import java.util.Map; |
|
18 |
|
|
19 |
/** |
|
20 |
* Created by antleb on 2/4/14. |
|
21 |
*/ |
|
22 |
public class SolrResultSetFactory implements ResultSetFactory { |
|
23 |
|
|
24 |
private Logger logger = Logger.getLogger(getClass()); |
|
25 |
//private Map<String, CloudSolrClient> clients = new HashMap<String, CloudSolrClient>(); |
|
26 |
|
|
27 |
@Autowired |
|
28 |
private PrometheusMeterRegistry prometheusMeterRegistry; |
|
29 |
|
|
30 |
private CloudSolrClient solrClient; |
|
31 |
private CloudSolrClient solrClientWithSocketTimeout; |
|
32 |
|
|
33 |
public void init(String[] zkservers, int socketTimeout) { |
|
34 |
logger.info("Initialization of clients."); |
|
35 |
|
|
36 |
logger.info("solrClient is set without socket timeout."); |
|
37 |
solrClient = new CloudSolrClient.Builder().withZkHost(Arrays.asList(zkservers)).build(); |
|
38 |
if (socketTimeout!=-1) { |
|
39 |
logger.info("solrClientWithSocketTimeout is set with socket timeout " + socketTimeout ); |
|
40 |
solrClientWithSocketTimeout = new CloudSolrClient.Builder().withZkHost(Arrays.asList(zkservers)) |
|
41 |
.withSocketTimeout(socketTimeout).build(); |
|
42 |
} else { //sudo solrClientWithSocketTimeout |
|
43 |
logger.info("solrClientWithSocketTimeout has ΝΟ actual timeout as the socketTimeout property is set to -1."); |
|
44 |
solrClientWithSocketTimeout = new CloudSolrClient.Builder().withZkHost(Arrays.asList(zkservers)).build(); |
|
45 |
} |
|
46 |
|
|
47 |
} |
|
48 |
|
|
49 |
public ResultSet<String> createResultSetWithSocketTimeout(EPR epr) throws IOException, CQLParseException, SearchServiceException { |
|
50 |
logger.info("Get RS with solrClient With SocketTimeout for EPR: " + epr); |
|
51 |
return new SolrResultSet(epr, solrClientWithSocketTimeout, prometheusMeterRegistry); |
|
52 |
} |
|
53 |
|
|
54 |
|
|
55 |
public ResultSet<String> createResultSetWithoutSocketTimeout(EPR epr) throws IOException, CQLParseException, SearchServiceException { |
|
56 |
logger.info("Get RS with solrClient Without SocketTimeout for EPR: " + epr); |
|
57 |
return new SolrResultSet(epr, solrClient, prometheusMeterRegistry); |
|
58 |
|
|
59 |
} |
|
60 |
|
|
61 |
@Override |
|
62 |
public ResultSet<String> createResultSet(EPR epr) { |
|
63 |
logger.info("Get RS with solrClient Without SocketTimeout for EPR: " + epr); |
|
64 |
try { |
|
65 |
return new SolrResultSet(epr, solrClient, prometheusMeterRegistry); |
|
66 |
|
|
67 |
} catch (IOException ioe) { |
|
68 |
logger.error("Error in search. I will return null resultSet.", ioe); |
|
69 |
|
|
70 |
} catch (CQLParseException cqlpe) { |
|
71 |
logger.error("Error in cql parse. I will return null resultSet.", cqlpe); |
|
72 |
|
|
73 |
} catch (SearchServiceException sse) { |
|
74 |
logger.error("Error in search.", sse); |
|
75 |
} |
|
76 |
|
|
77 |
return null; |
|
78 |
|
|
79 |
} |
|
80 |
|
|
81 |
@Override |
|
82 |
public <D> ResultSet<D> createResultSet(EPR epr, Class<D> resourceClass) { |
|
83 |
return (ResultSet<D>) createResultSet(epr); |
|
84 |
} |
|
85 |
|
|
86 |
@Override |
|
87 |
public <D> ResultSet<D> createResultSet(ResultSet<?> rs, Class<D> resourceClass) { |
|
88 |
throw new UnsupportedOperationException(); |
|
89 |
} |
|
90 |
|
|
91 |
@Override |
|
92 |
public <D extends SecureDriverResource> ResultSet<D> createSecurityAwareRS(EPR epr, Class<D> resourceClass) { |
|
93 |
throw new UnsupportedOperationException(); |
|
94 |
} |
|
95 |
} |
modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/micrometer/Metrics.java | ||
---|---|---|
1 |
package eu.dnetlib.data.search.micrometer; |
|
2 |
|
|
3 |
import io.micrometer.core.instrument.Tags; |
|
4 |
import io.micrometer.core.instrument.Timer; |
|
5 |
import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics; |
|
6 |
import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics; |
|
7 |
import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; |
|
8 |
import io.micrometer.core.instrument.binder.system.UptimeMetrics; |
|
9 |
import io.micrometer.prometheus.PrometheusMeterRegistry; |
|
10 |
import org.apache.log4j.Logger; |
|
11 |
import org.apache.maven.model.Dependency; |
|
12 |
import org.apache.maven.model.Model; |
|
13 |
import org.apache.maven.model.io.xpp3.MavenXpp3Reader; |
|
14 |
import org.codehaus.plexus.util.xml.pull.XmlPullParserException; |
|
15 |
import org.springframework.beans.factory.annotation.Autowired; |
|
16 |
import org.springframework.context.annotation.Configuration; |
|
17 |
import org.springframework.stereotype.Component; |
|
18 |
|
|
19 |
import javax.annotation.PostConstruct; |
|
20 |
import javax.ws.rs.core.Application; |
|
21 |
import java.io.IOException; |
|
22 |
import java.io.InputStreamReader; |
|
23 |
import java.time.Duration; |
|
24 |
import java.util.List; |
|
25 |
import java.util.concurrent.atomic.AtomicInteger; |
|
26 |
|
|
27 |
|
|
28 |
@Configuration |
|
29 |
@Component |
|
30 |
public class Metrics { |
|
31 |
|
|
32 |
private static final Logger logger = Logger.getLogger(Metrics.class); |
|
33 |
|
|
34 |
private static final AtomicInteger n = new AtomicInteger(1); |
|
35 |
|
|
36 |
@Autowired |
|
37 |
private PrometheusMeterRegistry registry; |
|
38 |
|
|
39 |
@PostConstruct |
|
40 |
public void init() { |
|
41 |
|
|
42 |
new JvmMemoryMetrics().bindTo(registry); |
|
43 |
new JvmGcMetrics().bindTo(registry); |
|
44 |
new JvmThreadMetrics().bindTo(registry); |
|
45 |
new UptimeMetrics().bindTo(registry); |
|
46 |
|
|
47 |
Timer.builder("search.server.response.duration").description("search response time with solr request and transformation included") |
|
48 |
.sla(Duration.ofMillis(100), Duration.ofMillis(500), Duration.ofMillis(1000), Duration.ofMillis(2000), Duration.ofMillis(2500), |
|
49 |
Duration.ofMillis(3000),Duration.ofMillis(4000), Duration.ofMillis(5000)).register(registry); |
|
50 |
|
|
51 |
Timer.builder("solr.server.response.duration").description("solr response time") |
|
52 |
.sla(Duration.ofMillis(100), Duration.ofMillis(500), Duration.ofMillis(1000), Duration.ofMillis(2000), Duration.ofMillis(2500), |
|
53 |
Duration.ofMillis(3000),Duration.ofMillis(4000), Duration.ofMillis(5000)).register(registry); |
|
54 |
|
|
55 |
MavenXpp3Reader reader = new MavenXpp3Reader(); |
|
56 |
Model model = null; |
|
57 |
|
|
58 |
try { |
|
59 |
model = reader.read( |
|
60 |
new InputStreamReader( |
|
61 |
Application.class.getResourceAsStream( |
|
62 |
"/META-INF/maven/eu.dnetlib/uoa-search/pom.xml" |
|
63 |
) |
|
64 |
) |
|
65 |
); |
|
66 |
|
|
67 |
} catch (XmlPullParserException e) { |
|
68 |
logger.error("Error reading pom. ", e); |
|
69 |
|
|
70 |
} catch (IOException e) { |
|
71 |
logger.error("Error reading pom. ", e); |
|
72 |
} { |
|
73 |
|
|
74 |
} |
|
75 |
|
|
76 |
List<Dependency> list = model.getDependencies(); |
|
77 |
Dependency micrometer = list.stream() |
|
78 |
.filter(dependency -> dependency.getArtifactId().equals("micrometer-registry-prometheus")) |
|
79 |
.findFirst() |
|
80 |
.get(); |
|
81 |
|
|
82 |
registry.gauge("micrometer_info", Tags.of("component", model.getArtifactId()).and("scmtag", "HEAD").and("version", model.getVersion()), n); |
|
83 |
//registry.summary("search.server.response.size"); |
|
84 |
|
|
85 |
} |
|
86 |
} |
modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/web/api/MicrometerPublisher.java | ||
---|---|---|
1 |
package eu.dnetlib.data.search.web.api; |
|
2 |
|
|
3 |
import io.micrometer.prometheus.PrometheusMeterRegistry; |
|
4 |
import org.springframework.beans.factory.annotation.Autowired; |
|
5 |
import org.springframework.context.annotation.ComponentScan; |
|
6 |
import org.springframework.http.MediaType; |
|
7 |
import org.springframework.stereotype.Component; |
|
8 |
import org.springframework.stereotype.Controller; |
|
9 |
import org.springframework.web.bind.annotation.RequestMapping; |
|
10 |
import org.springframework.web.bind.annotation.RequestMethod; |
|
11 |
import org.springframework.web.bind.annotation.ResponseBody; |
|
12 |
|
|
13 |
import javax.servlet.http.HttpServletRequest; |
|
14 |
import javax.servlet.http.HttpServletResponse; |
|
15 |
|
|
16 |
@Controller |
|
17 |
@ComponentScan |
|
18 |
@Component |
|
19 |
public class MicrometerPublisher { |
|
20 |
|
|
21 |
@Autowired |
|
22 |
PrometheusMeterRegistry registry; |
|
23 |
|
|
24 |
@RequestMapping(method = RequestMethod.GET, value = "/metrics", produces = {MediaType.TEXT_PLAIN_VALUE}) |
|
25 |
@ResponseBody |
|
26 |
public String getMetrics(HttpServletRequest request, HttpServletResponse response) throws Exception { |
|
27 |
return registry.scrape(); |
|
28 |
} |
|
29 |
} |
modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/web/api/TSVResponseFormat.java | ||
---|---|---|
1 |
package eu.dnetlib.data.search.web.api; |
|
2 |
|
|
3 |
/** |
|
4 |
* Created by kiatrop on 20/12/2016. |
|
5 |
*/ |
|
6 |
public class TSVResponseFormat { |
|
7 |
public static final String organization = "Name\tProjects\tCountry\n"; |
|
8 |
public static final String response = "%s%s"; |
|
9 |
} |
modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/web/api/HTMLResponseFormat.java | ||
---|---|---|
1 |
package eu.dnetlib.data.search.web.api; |
|
2 |
|
|
3 |
/** |
|
4 |
* Created by kiatrop on 16/1/2017. |
|
5 |
*/ |
|
6 |
public class HTMLResponseFormat { |
|
7 |
public static final String response = "<table>" + |
|
8 |
"<thead><tr>" + |
|
9 |
" <th>Title</th>" + |
|
10 |
"<th>Authors</th>" + |
|
11 |
"<th>Publication Year</th>" + |
|
12 |
"<th>DOI</th>\n" + |
|
13 |
"<th>Permanent Identifier</th>" + |
|
14 |
"<th>Publication type</th>" + |
|
15 |
"<th>Journal</th>" + |
|
16 |
"<th>Project Name (GA Number)</th>" + |
|
17 |
"<th>Access Mode</th>" + |
|
18 |
"</tr>" + |
|
19 |
"</thead>" + |
|
20 |
"<tbody>%s</tbody></table>"; |
|
21 |
} |
modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/web/SearchServiceContextUtils.java | ||
---|---|---|
1 |
/** |
|
2 |
* |
|
3 |
*/ |
|
4 |
package eu.dnetlib.data.search.web; |
|
5 |
|
|
6 |
import eu.dnetlib.api.data.SearchService; |
|
7 |
import gr.uoa.di.driver.enabling.resultset.ResultSetFactory; |
|
8 |
import org.springframework.context.ApplicationContext; |
|
9 |
import org.springframework.web.context.support.WebApplicationContextUtils; |
|
10 |
import org.w3c.dom.Document; |
|
11 |
import org.w3c.dom.Element; |
|
12 |
import org.xml.sax.InputSource; |
|
13 |
import org.xml.sax.SAXException; |
|
14 |
|
|
15 |
import javax.servlet.ServletContext; |
|
16 |
import javax.servlet.http.HttpServletRequest; |
|
17 |
import javax.xml.parsers.DocumentBuilder; |
|
18 |
import javax.xml.parsers.DocumentBuilderFactory; |
|
19 |
import javax.xml.parsers.ParserConfigurationException; |
|
20 |
import javax.xml.transform.*; |
|
21 |
import javax.xml.transform.dom.DOMSource; |
|
22 |
import javax.xml.transform.stream.StreamResult; |
|
23 |
import java.io.IOException; |
|
24 |
import java.io.StringReader; |
|
25 |
import java.io.StringWriter; |
|
26 |
|
|
27 |
/** |
|
28 |
* @author stoumpos |
|
29 |
* |
|
30 |
*/ |
|
31 |
public class SearchServiceContextUtils { |
|
32 |
|
|
33 |
public static SearchService getSearchService(HttpServletRequest request) { |
|
34 |
ServletContext session = request.getSession().getServletContext(); |
|
35 |
ApplicationContext context = WebApplicationContextUtils |
|
36 |
.getWebApplicationContext(session); |
|
37 |
SearchService search = (SearchService) context.getBean("searchService"); |
|
38 |
|
|
39 |
return search; |
|
40 |
} |
|
41 |
|
|
42 |
public static ResultSetFactory getResultSetFactory( |
|
43 |
HttpServletRequest request) { |
|
44 |
ServletContext session = request.getSession().getServletContext(); |
|
45 |
ApplicationContext context = WebApplicationContextUtils |
|
46 |
.getWebApplicationContext(session); |
|
47 |
ResultSetFactory rsFactory = (ResultSetFactory) context |
|
48 |
.getBean("resultSetFactory"); |
|
49 |
|
|
50 |
return rsFactory; |
|
51 |
} |
|
52 |
|
|
53 |
public static final String prettyPrint(String xml) |
|
54 |
throws TransformerConfigurationException, |
|
55 |
TransformerFactoryConfigurationError, TransformerException, |
|
56 |
SAXException, IOException, ParserConfigurationException { |
|
57 |
|
|
58 |
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); |
|
59 |
dbf.setValidating(true); |
|
60 |
DocumentBuilder db = dbf.newDocumentBuilder(); |
|
61 |
InputSource sourceXML = new InputSource(new StringReader(xml)); |
|
62 |
Document xmlDoc = db.parse(sourceXML); |
|
63 |
Element source = xmlDoc.getDocumentElement(); |
|
64 |
source.normalize(); |
|
65 |
|
|
66 |
StreamResult target = new StreamResult(new StringWriter()); |
|
67 |
|
|
68 |
Transformer tf = TransformerFactory.newInstance().newTransformer(); |
|
69 |
tf.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); |
|
70 |
tf.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); |
|
71 |
tf.setOutputProperty(OutputKeys.INDENT, "yes"); |
|
72 |
tf.transform(new DOMSource(source), target); |
|
73 |
|
|
74 |
return target.getWriter().toString(); |
|
75 |
} |
|
76 |
|
|
77 |
} |
|
0 | 78 |
modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/micrometer/MicrometerComponent.java | ||
---|---|---|
1 |
package eu.dnetlib.data.search.micrometer; |
|
2 |
|
|
3 |
import io.micrometer.core.aop.TimedAspect; |
|
4 |
import io.micrometer.core.instrument.Clock; |
|
5 |
import io.micrometer.core.instrument.MeterRegistry; |
|
6 |
import io.micrometer.core.instrument.Tag; |
|
7 |
import io.micrometer.core.instrument.Tags; |
|
8 |
import io.micrometer.prometheus.PrometheusConfig; |
|
9 |
import io.micrometer.prometheus.PrometheusMeterRegistry; |
|
10 |
import io.micrometer.spring.web.servlet.WebMvcTags; |
|
11 |
import io.micrometer.spring.web.servlet.WebMvcTagsProvider; |
|
12 |
import io.prometheus.client.CollectorRegistry; |
|
13 |
import org.springframework.context.annotation.Bean; |
|
14 |
import org.springframework.context.annotation.ComponentScan; |
|
15 |
import org.springframework.context.annotation.Configuration; |
|
16 |
import org.springframework.stereotype.Component; |
|
17 |
import org.springframework.web.servlet.config.annotation.EnableWebMvc; |
|
18 |
|
|
19 |
import javax.servlet.http.HttpServletRequest; |
|
20 |
import javax.servlet.http.HttpServletResponse; |
|
21 |
import java.util.ArrayList; |
|
22 |
|
|
23 |
@ComponentScan |
|
24 |
@Configuration |
|
25 |
@Component |
|
26 |
@EnableWebMvc |
|
27 |
public class MicrometerComponent { |
|
28 |
|
|
29 |
@Bean |
|
30 |
public Clock micrometerClock() { |
|
31 |
return Clock.SYSTEM; |
|
32 |
} |
|
33 |
|
|
34 |
@Bean |
|
35 |
public PrometheusConfig prometheusConfig() { |
|
36 |
return PrometheusConfig.DEFAULT; |
|
37 |
} |
|
38 |
|
|
39 |
@Bean |
|
40 |
public PrometheusMeterRegistry prometheusMeterRegistry(PrometheusConfig config, CollectorRegistry collectorRegistry, |
|
41 |
Clock clock) { |
|
42 |
return new PrometheusMeterRegistry(config, collectorRegistry, clock); |
|
43 |
} |
|
44 |
|
|
45 |
@Bean |
|
46 |
public CollectorRegistry collectorRegistry() { |
|
47 |
return new CollectorRegistry(true); |
|
48 |
} |
|
49 |
|
|
50 |
@Bean |
|
51 |
public TimedAspect timedAspect(MeterRegistry registry) { |
|
52 |
return new TimedAspect(registry); |
|
53 |
} |
|
54 |
|
|
55 |
@Bean |
|
56 |
public WebMvcTagsProvider webMvcTagsProvider() { |
|
57 |
return new WebMvcTagsProvider() { |
|
58 |
@Override |
|
59 |
public Iterable<Tag> httpLongRequestTags(HttpServletRequest httpServletRequest, Object o) { |
|
60 |
return new ArrayList<>(); |
|
61 |
} |
|
62 |
|
|
63 |
@Override |
|
64 |
public Iterable<Tag> httpRequestTags(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, Object o, Throwable throwable) { |
|
65 |
Tag uriTag = Tag.of("status", String.valueOf(httpServletResponse.getStatus())); |
|
66 |
return Tags.of(WebMvcTags.method(httpServletRequest), uriTag, WebMvcTags.exception(throwable), WebMvcTags.status(httpServletResponse)); |
|
67 |
} |
|
68 |
}; |
|
69 |
} |
|
70 |
|
|
71 |
/*@Bean |
|
72 |
public MappedInterceptor myInterceptor(PrometheusMeterRegistry prometheusMeterRegistry) { |
|
73 |
return new MappedInterceptor(null, new PrometheusRequestTimerInterceptor(prometheusMeterRegistry)); |
|
74 |
}*/ |
|
75 |
|
|
76 |
} |
modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/transform/config/SearchRegistry.java | ||
---|---|---|
1 |
package eu.dnetlib.data.search.transform.config; |
|
2 |
|
|
3 |
|
|
4 |
import eu.dnetlib.data.search.transform.*; |
|
5 |
import eu.dnetlib.data.search.transform.formatter.Formatter; |
|
6 |
import eu.dnetlib.data.search.transform.formatter.XSLTFormatter; |
|
7 |
import eu.dnetlib.data.search.transform.utils.VelocityUtil; |
|
8 |
import eu.dnetlib.data.search.utils.vocabulary.VocabularyManager; |
|
9 |
import org.apache.commons.io.IOUtils; |
|
10 |
import org.apache.log4j.Logger; |
|
11 |
import org.apache.velocity.Template; |
|
12 |
import org.apache.velocity.VelocityContext; |
|
13 |
import org.apache.velocity.app.VelocityEngine; |
|
14 |
|
|
15 |
import java.io.InputStream; |
|
16 |
import java.io.StringWriter; |
|
17 |
import java.util.*; |
|
18 |
|
|
19 |
public class SearchRegistry implements TransformerRegistry, FormatterRegistry { |
|
20 |
private Logger logger = Logger.getLogger(this.getClass()); |
|
21 |
|
|
22 |
private Configuration config = null; |
|
23 |
private VocabularyManager vocabularyManager = null; |
|
24 |
|
|
25 |
private Map<Locale,HashMap<String,Transformer>> transformers = new HashMap<Locale, HashMap<String,Transformer>>(); |
|
26 |
private Map<String, Formatter> formatters = new HashMap<String, Formatter>(); |
|
27 |
|
|
28 |
@Override |
|
29 |
public Transformer getTransformer(String name, Locale locale) { |
|
30 |
logger.debug("Requested transformer " + name + " locale " + locale); |
|
31 |
Transformer tx = null; |
|
32 |
|
|
33 |
if(name == null) { |
|
34 |
return null; |
|
35 |
} |
|
36 |
|
|
37 |
if (locale == null) { |
|
38 |
locale = config.getDefaultLocale(); |
|
39 |
} |
|
40 |
|
|
41 |
try { |
|
42 |
if (this.transformers.get(locale)!= null) { |
|
43 |
tx = this.transformers.get(locale).get(name); |
|
44 |
|
|
45 |
} else { //not map for locale |
|
46 |
logger.debug("Transformation map " + config.getTransformationsMap().keySet()); |
|
47 |
tx = createChainTransformer(name, locale, config.getTransformationsMap().get(name)); |
|
48 |
Map<String, Transformer> map = new HashMap<String, Transformer>(); |
|
49 |
map.put(name, tx); |
|
50 |
transformers.put(locale, (HashMap<String, Transformer>) map); |
|
51 |
} |
|
52 |
|
|
53 |
if (tx == null) { // there is the local map but not the transformer |
|
54 |
tx = createChainTransformer(name, locale, config.getTransformationsMap().get(name)); |
|
55 |
transformers.get(locale).put(name, tx); |
|
56 |
} |
|
57 |
|
|
58 |
} catch (Exception e) { |
|
59 |
logger.error("Cannot create transformer " + name, e); |
|
60 |
throw new IllegalArgumentException("Cannot create transformer " + name, e); |
|
61 |
} |
|
62 |
|
|
63 |
logger.debug("Returned in get transformer" + tx); |
|
64 |
return tx; |
|
65 |
} |
|
66 |
|
|
67 |
@Override |
|
68 |
public Formatter getFormatter(String name) { |
|
69 |
Formatter fm = this.formatters.get(name); |
|
70 |
|
|
71 |
try { |
|
72 |
if (fm == null) { |
|
73 |
FormatterConfiguration formatterConfiguration = config.getFormattersMap().get(name); |
|
74 |
String formatterXslt = formatterConfiguration.getXsl_file(); |
|
75 |
String template = formatterConfiguration.getTemplate(); |
|
76 |
|
|
77 |
logger.debug("Configuration for formatter: " + formatterXslt + ", " + template); |
|
78 |
|
|
79 |
if (formatterXslt == null) { |
|
80 |
logger.error("Cannot create formatter " + name); |
|
81 |
throw new IllegalArgumentException("Cannot create formatter " + name); |
|
82 |
} |
|
83 |
|
|
84 |
if (template == null) { |
|
85 |
fm = createFormatter(formatterXslt); |
|
86 |
} else { |
|
87 |
fm = createFormatter(formatterXslt, template); |
|
88 |
} |
|
89 |
|
|
90 |
formatters.put(name, fm); |
|
91 |
} |
|
92 |
|
|
93 |
} catch (Exception e) { |
|
94 |
logger.error("Cannot create formatter " + name); |
|
95 |
throw new IllegalArgumentException("Cannot create formatter " + name, e); |
|
96 |
} |
|
97 |
|
|
98 |
return fm; |
|
99 |
} |
|
100 |
|
|
101 |
private Transformer createChainTransformer(String name, Locale locale, List<Transformation> transformations) throws SearchFactoryException { |
|
102 |
List<XPathTrasformation> temp = new ArrayList<XPathTrasformation>(); |
|
103 |
List<Transformer> forChaining = new ArrayList<Transformer>(); |
|
104 |
for (Transformation tr: transformations){ |
|
105 |
if (tr instanceof XSLTTransformation) { |
|
106 |
try { |
|
107 |
if (!temp.isEmpty()){ |
|
108 |
forChaining.add(createTransformer(locale, temp)); |
|
109 |
} |
|
110 |
temp.clear(); |
|
111 |
forChaining.add(createTransformer((XSLTTransformation)tr)); |
|
112 |
} catch (Exception e) { |
|
113 |
logger.warn("Error creating a transformer with name: "+name+" and locale: "+locale.getDisplayName(), e); |
|
114 |
throw new SearchFactoryException("Error creating a transformer with name: "+name+" and locale: "+locale.getDisplayName(), e); |
|
115 |
|
|
116 |
} |
|
117 |
} |
|
118 |
else { |
|
119 |
temp.add((XPathTrasformation)tr); |
|
120 |
} |
|
121 |
} |
|
122 |
if (!temp.isEmpty()){ |
|
123 |
try { |
|
124 |
forChaining.add(createTransformer(locale, temp)); |
|
125 |
} catch (Exception e) { |
|
126 |
logger.warn("Error creating a transformer with name: " + name + " and locale: "+locale.getDisplayName(), e); |
|
127 |
throw new SearchFactoryException("Error creating a transformer with name: " + name + " and locale: "+locale.getDisplayName(), e); |
|
128 |
} |
|
129 |
} |
|
130 |
return new ChainTransformer(name, forChaining); |
|
131 |
} |
|
132 |
|
|
133 |
private Transformer createTransformer(XSLTTransformation transformation) throws Exception { |
|
134 |
String xsl_file = transformation.getXslt(); |
|
135 |
InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(xsl_file); |
|
136 |
StringWriter writer = new StringWriter(); |
|
137 |
IOUtils.copy(inputStream, writer); |
|
138 |
String xsl = writer.toString(); |
|
139 |
|
|
140 |
XsltTransformer OptimusPrime = new XsltTransformer(xsl); |
|
141 |
|
|
142 |
writer.close(); |
|
143 |
IOUtils.closeQuietly(inputStream); |
|
144 |
|
|
145 |
return OptimusPrime; |
|
146 |
} |
|
147 |
|
|
148 |
private Transformer createTransformer(Locale locale, List<XPathTrasformation> transformations) throws Exception { |
|
149 |
VelocityEngine ve = VelocityUtil.getEngine(); |
|
150 |
|
|
151 |
Template t = ve.getTemplate("/eu/dnetlib/data/search/transform/config/velocity.vm", "UTF-8"); |
|
152 |
VelocityContext context = new VelocityContext(); |
|
153 |
|
|
154 |
context.put("locale", locale); |
|
155 |
context.put("defLocale", config.getDefaultLocale()); |
|
156 |
context.put("transformations", transformations); |
|
157 |
StringWriter writer = new StringWriter(); |
|
158 |
t.merge(context, writer); |
|
159 |
XsltTransformer Megatron = new XsltTransformer(writer.toString()); |
|
160 |
|
|
161 |
//logger.debug("Transformer xsl " + Megatron.getXslt()); |
|
162 |
//logger.debug("Vocab paths are: " + loadVocabulariesPathMap(transformations, locale).toString()); |
|
163 |
|
|
164 |
return Megatron; |
|
165 |
} |
|
166 |
|
|
167 |
private Formatter createFormatter(String xsl_file) throws Exception { |
|
168 |
logger.debug("Loading xslt " + xsl_file); |
|
169 |
InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(xsl_file); |
|
170 |
|
|
171 |
StringWriter writer = new StringWriter(); |
|
172 |
IOUtils.copy(inputStream, writer); |
|
173 |
String xsl = writer.toString(); |
|
174 |
|
|
175 |
XSLTFormatter Cybertron = new XSLTFormatter(xsl); |
|
176 |
writer.close(); |
|
177 |
IOUtils.closeQuietly(inputStream); |
|
178 |
|
|
179 |
return Cybertron; |
|
180 |
} |
|
181 |
|
|
182 |
private Formatter createFormatter(String xsl_file, String template) throws Exception { |
|
183 |
logger.debug("Loading xslt " + xsl_file); |
|
184 |
logger.debug("Loading template " + template); |
|
185 |
|
|
186 |
InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(xsl_file); |
|
187 |
|
|
188 |
StringWriter writer = new StringWriter(); |
|
189 |
IOUtils.copy(inputStream, writer); |
|
190 |
String xsl = writer.toString(); |
|
191 |
|
|
192 |
XSLTFormatter Cybertron = new XSLTFormatter(xsl, template); |
|
193 |
writer.close(); |
|
194 |
IOUtils.closeQuietly(inputStream); |
|
195 |
|
|
196 |
return Cybertron; |
|
197 |
} |
|
198 |
|
|
199 |
public Configuration getConfig() { |
|
200 |
return config; |
|
201 |
} |
|
202 |
|
|
203 |
public void setConfig(Configuration config) { |
|
204 |
this.config = config; |
|
205 |
} |
|
206 |
|
|
207 |
public void setVocabularyManager(VocabularyManager vocabularyManager) { |
|
208 |
this.vocabularyManager = vocabularyManager; |
|
209 |
} |
|
210 |
|
|
211 |
public VocabularyManager getVocabularyManager() { |
|
212 |
return vocabularyManager; |
|
213 |
} |
|
214 |
|
|
215 |
} |
modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/app/plan/FieldRewriteRule.java | ||
---|---|---|
1 |
package eu.dnetlib.data.search.app.plan; |
|
2 |
|
|
3 |
/** |
|
4 |
*/ |
|
5 |
public abstract class FieldRewriteRule { |
|
6 |
|
|
7 |
protected String name = null; |
|
8 |
protected String fieldName = null; |
|
9 |
|
|
10 |
public FieldRewriteRule(String name, String fieldName) { |
|
11 |
this.name = name; |
|
12 |
this.fieldName = fieldName; |
|
13 |
} |
|
14 |
|
|
15 |
public String getFieldName() { |
|
16 |
return fieldName; |
|
17 |
} |
|
18 |
|
|
19 |
public void setFieldName(String fieldName) { |
|
20 |
this.fieldName = fieldName; |
|
21 |
} |
|
22 |
|
|
23 |
public String getName() { |
|
24 |
return name; |
|
25 |
} |
|
26 |
|
|
27 |
public void setName(String name) { |
|
28 |
this.name = name; |
|
29 |
} |
|
30 |
|
|
31 |
//public abstract CqlClause apply(CqlRelation relation) throws CqlException; |
|
32 |
|
|
33 |
@Override |
|
34 |
public String toString() { |
|
35 |
return name; |
|
36 |
} |
|
37 |
} |
modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/web/api/ExceptionController.java | ||
---|---|---|
1 |
package eu.dnetlib.data.search.web.api; |
|
2 |
|
|
3 |
|
|
4 |
import io.micrometer.prometheus.PrometheusMeterRegistry; |
|
5 |
import org.springframework.beans.factory.annotation.Autowired; |
|
6 |
import org.springframework.http.HttpStatus; |
|
7 |
import org.springframework.http.ResponseEntity; |
|
8 |
import org.springframework.web.bind.annotation.ControllerAdvice; |
|
9 |
import org.springframework.web.bind.annotation.ExceptionHandler; |
|
10 |
import org.springframework.web.servlet.config.annotation.EnableWebMvc; |
|
11 |
import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; |
|
12 |
|
|
13 |
@EnableWebMvc |
|
14 |
@ControllerAdvice(value = "eu.dnetlib.data.search.web.api", assignableTypes = SearchRequestController.class) |
|
15 |
public class ExceptionController extends ResponseEntityExceptionHandler{ |
|
16 |
|
|
17 |
@Autowired |
|
18 |
PrometheusMeterRegistry registry; |
|
19 |
|
|
20 |
@ExceptionHandler(Exception.class) |
|
21 |
public ResponseEntity<Error> invalidInput(Exception ex) { |
|
22 |
Error response = new Error(); |
|
23 |
response.setStatus("error"); |
|
24 |
response.setCode("500"); |
|
25 |
response.setMessage(ex.getMessage()); |
|
26 |
response.setException(ex.getCause().getMessage()); |
|
27 |
|
|
28 |
|
|
29 |
registry.counter("http.status.500", "500", "uri").increment(); |
|
30 |
return new ResponseEntity<Error>(response, HttpStatus.CONFLICT); |
|
31 |
} |
|
32 |
} |
modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/web/api/CSVResponseFormat.java | ||
---|---|---|
1 |
package eu.dnetlib.data.search.web.api; |
|
2 |
|
|
3 |
import eu.dnetlib.data.search.web.utils.RequestResponseHandler; |
|
4 |
import org.apache.log4j.Logger; |
|
5 |
|
|
6 |
/** |
|
7 |
* Created by kiatrop on 20/12/2016. |
|
8 |
*/ |
|
9 |
public class CSVResponseFormat { |
|
10 |
|
|
11 |
public static final String organization = "Name,Projects,Country\r\n"; |
|
12 |
public static final String datasource = "Name,Type,Compatibility,OAI-PMH,website\r\n"; |
|
13 |
public static final String service = "Name,website\r\n"; |
|
14 |
public static final String person = "Fullname\r\n"; |
|
15 |
public static final String project = "Title,Funder,Start Date,End Date,Participants\r\n"; |
|
16 |
// public static final String result = "Type,Title,Authors,Publication Year,DOI,Green,OA colour,Download from,Type,Diamond,Journal,Publicly Funded,Funder|Project Name (GA Number),Access\r\n"; |
|
17 |
public static final String result = "Type,Title,Authors,Publication Year,DOI,Type,Peer Reviewed,Diamond Journal,Journal,Download from,Publicly Funded,Funder|Project Name (GA Number),Access,Green,OA colour\r\n"; |
|
18 |
|
|
19 |
//TODO SPECIAL |
|
20 |
public static final String result_special = "Type,Title,Authors,Publication Year,DOI,Type,Peer Reviewed,Diamond Journal,Journal,Download from,Publicly Funded,Funder,Project Name (GA Number),Access,Green,OA colour\r\n"; |
|
21 |
|
|
22 |
private static Logger logger = Logger.getLogger(CSVResponseFormat.class); |
|
23 |
|
|
24 |
|
|
25 |
public static final String response = "%s%s"; |
|
26 |
|
|
27 |
public static String appendTitle(RequestResponseHandler.Entity entity, boolean special) { |
|
28 |
if(entity.equals(RequestResponseHandler.Entity.ORGANIZATION)) { |
|
29 |
return organization; |
|
30 |
|
|
31 |
} else if (entity.equals(RequestResponseHandler.Entity.DATASOURCE)) { |
|
32 |
return datasource; |
|
33 |
|
|
34 |
} else if (entity.equals(RequestResponseHandler.Entity.SERVICE)) { |
|
35 |
return service; |
|
36 |
} |
|
37 |
/* else if (entity.equals(RequestResponseHandler.Entity.PERSON)) { |
|
38 |
return person; |
|
39 |
|
|
40 |
}*/ else if (entity.equals(RequestResponseHandler.Entity.PROJECT)) { |
|
41 |
return project; |
|
42 |
|
|
43 |
} else if (entity.equals(RequestResponseHandler.Entity.PUBLICATION) || entity.equals(RequestResponseHandler.Entity.DATASET) |
|
44 |
||entity.equals(RequestResponseHandler.Entity.SOFTWARE) || entity.equals(RequestResponseHandler.Entity.OTHER) |
|
45 |
|| entity.equals(RequestResponseHandler.Entity.RESULT)) { |
|
46 |
|
|
47 |
if (special) { |
|
48 |
return result_special; |
|
49 |
} |
|
50 |
|
|
51 |
return result; |
|
52 |
} |
|
53 |
|
|
54 |
return ""; |
|
55 |
} |
|
56 |
|
|
57 |
|
|
58 |
|
|
59 |
} |
modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/transform/config/vocabularies/FormatterParams.java | ||
---|---|---|
1 |
package eu.dnetlib.data.search.transform.config.vocabularies; |
|
2 |
|
|
3 |
/** |
|
4 |
* Created by kiatrop on 18/6/2014. |
|
5 |
*/ |
|
6 |
public class FormatterParams { |
|
7 |
String xsl_file; |
|
8 |
String template_file; |
|
9 |
|
|
10 |
public String getXsl_file() { |
|
11 |
return xsl_file; |
|
12 |
} |
|
13 |
|
|
14 |
public void setXsl_file(String xsl_file) { |
|
15 |
this.xsl_file = xsl_file; |
|
16 |
} |
|
17 |
|
|
18 |
public String getTemplate_file() { |
|
19 |
return template_file; |
|
20 |
} |
|
21 |
|
|
22 |
public void setTemplate_file(String template_file) { |
|
23 |
this.template_file = template_file; |
|
24 |
} |
|
25 |
} |
modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/transform/config/XPathTrasformation.java | ||
---|---|---|
1 |
package eu.dnetlib.data.search.transform.config; |
|
2 |
|
|
3 |
import java.util.ArrayList; |
|
4 |
import java.util.Arrays; |
|
5 |
import java.util.List; |
|
6 |
|
|
7 |
public class XPathTrasformation extends Transformation { |
|
8 |
private String change; |
|
9 |
private String vocabulary; |
|
10 |
private List<String> matches; |
|
11 |
|
|
12 |
public XPathTrasformation(String match, String change, String vocabulary) { |
|
13 |
this.change = change; |
|
14 |
this.vocabulary = vocabulary; |
|
15 |
this.matches = new ArrayList<String>(Arrays.asList(match.split("\\."))); |
|
16 |
} |
|
17 |
|
|
18 |
public List<String> getMatches() { |
|
19 |
return matches; |
|
20 |
} |
|
21 |
|
|
22 |
public String getChange() { |
|
23 |
return change; |
|
24 |
} |
|
25 |
|
|
26 |
public void setChange(String change) { |
|
27 |
this.change = change; |
|
28 |
} |
|
29 |
|
|
30 |
public String getVocabulary() { |
|
31 |
return vocabulary; |
|
32 |
} |
|
33 |
|
|
34 |
public void setVocabulary(String vocabulary) { |
|
35 |
this.vocabulary = vocabulary; |
|
36 |
} |
|
37 |
} |
modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/solr/SolrResultSet.java | ||
---|---|---|
1 |
package eu.dnetlib.data.search.solr; |
|
2 |
|
|
3 |
import com.google.gson.Gson; |
|
4 |
import eu.dnetlib.api.data.SearchServiceException; |
|
5 |
import eu.dnetlib.data.search.transform.Transformer; |
|
6 |
import eu.dnetlib.data.search.transform.TransformerException; |
|
7 |
import eu.dnetlib.data.search.utils.solr.SolrResultSetOptionsUtil; |
|
8 |
import eu.dnetlib.data.search.utils.solr.SolrResultsFormatter; |
|
9 |
import eu.dnetlib.domain.EPR; |
|
10 |
import gr.uoa.di.driver.enabling.resultset.ResultSet; |
|
11 |
import io.micrometer.core.instrument.Timer; |
|
12 |
import io.micrometer.prometheus.PrometheusMeterRegistry; |
|
13 |
import org.apache.commons.lang.StringEscapeUtils; |
|
14 |
import org.apache.log4j.Logger; |
|
15 |
import org.apache.solr.client.solrj.SolrClient; |
|
16 |
import org.apache.solr.client.solrj.SolrServerException; |
|
17 |
import org.apache.solr.client.solrj.impl.CloudSolrClient; |
|
18 |
import org.apache.solr.client.solrj.response.FacetField; |
|
19 |
import org.apache.solr.client.solrj.response.QueryResponse; |
|
20 |
import org.apache.solr.common.SolrDocumentList; |
|
21 |
import org.apache.solr.common.params.SolrParams; |
|
22 |
import org.apache.solr.common.util.NamedList; |
|
23 |
import org.z3950.zing.cql.CQLParseException; |
|
24 |
|
|
25 |
import javax.ws.rs.core.MediaType; |
|
26 |
import java.io.IOException; |
|
27 |
import java.io.OutputStream; |
|
28 |
import java.util.*; |
|
29 |
|
|
30 |
/** |
|
31 |
* Created by antleb on 2/4/14. |
|
32 |
*/ |
|
33 |
|
|
34 |
public class SolrResultSet implements ResultSet<String> { |
|
35 |
|
|
36 |
private Logger logger = Logger.getLogger(getClass()); |
|
37 |
|
|
38 |
private EPR epr = null; |
|
39 |
public SolrClient solrClient = null; |
|
40 |
|
|
41 |
private NamedList<String> queryOpts = new NamedList<String>(); |
|
42 |
long size = -1; |
|
43 |
|
|
44 |
private PrometheusMeterRegistry registry; |
|
45 |
|
|
46 |
public SolrResultSet(EPR epr, CloudSolrClient cloudSolrClient, PrometheusMeterRegistry registry) |
|
47 |
throws IOException, CQLParseException, SearchServiceException { |
|
48 |
this.epr = epr; |
|
49 |
this.solrClient = cloudSolrClient; |
|
50 |
|
|
51 |
this.queryOpts = SolrResultSetOptionsUtil.extractQueryOptions(epr.getParameter("query")); |
|
52 |
|
|
53 |
String layout = epr.getParameter("layout"); |
|
54 |
String mdformat = epr.getParameter("mdformat"); |
|
55 |
String interpretation = epr.getParameter("interpretation"); |
|
56 |
|
|
57 |
((CloudSolrClient)solrClient).setDefaultCollection(mdformat + "-" + layout + "-" + interpretation); |
|
58 |
|
|
59 |
this.registry = registry; |
|
60 |
} |
|
61 |
|
|
62 |
@Override |
|
63 |
public boolean isOpen() { |
|
64 |
return true; |
|
65 |
} |
|
66 |
|
|
67 |
@Override |
|
68 |
public boolean isAlive() { |
|
69 |
return true; |
|
70 |
} |
|
71 |
|
|
72 |
@Override |
|
73 |
public void close() { |
|
74 |
|
|
75 |
/* Closes the whole pull. We do not want this. |
|
76 |
try { |
|
77 |
logger.info("!!!!!!!!! !!!!!!! CLOSING !!!!!!!!! !!!!!!!!!! "); |
|
78 |
solrClient.close(); |
|
79 |
|
|
80 |
} catch (IOException e) { |
|
81 |
logger.error("Error closing result set.", e); |
|
82 |
}*/ |
|
83 |
} |
|
84 |
|
|
85 |
@Override |
|
86 |
public int size() { |
|
87 |
return (int) size; |
|
88 |
} |
|
89 |
|
|
90 |
@Override |
|
91 |
@Deprecated |
|
92 |
public List<String> getElements(int from, int to) { |
|
93 |
return get(from, to); |
|
94 |
} |
|
95 |
|
|
96 |
List<FacetField> facetFields = null; |
|
97 |
|
|
98 |
@Override |
|
99 |
@Deprecated |
|
100 |
public List<String> get(int from, int to) { |
|
101 |
List<String> res = new ArrayList<String>(); |
|
102 |
|
|
103 |
QueryResponse rsp = null; |
|
104 |
|
|
105 |
HashMap<String, List<String>> map = new HashMap<String, List<String>>(); |
|
106 |
|
|
107 |
logger.debug("from: " + from); |
|
108 |
logger.debug("to: " + to); |
|
109 |
|
|
110 |
|
|
111 |
queryOpts.add("start", from+1 + ""); |
|
112 |
queryOpts.add("rows", to + 1+""); |
|
113 |
|
|
114 |
try { |
|
115 |
|
|
116 |
rsp = solrClient.query(SolrParams.toSolrParams(queryOpts)); |
|
117 |
|
|
118 |
facetFields = rsp.getFacetFields(); |
|
119 |
SolrDocumentList docs = rsp.getResults(); |
|
120 |
|
|
121 |
if (facetFields!=null && !facetFields.isEmpty()) { |
|
122 |
for (int i = from - 1; i < to; i++) { |
|
123 |
for (FacetField field : facetFields) { |
|
124 |
if (field.getValueCount() > i) { |
|
125 |
BrowseField bf = new BrowseField(); |
|
126 |
bf.setId(field.getValues().get(i).getName()); |
|
127 |
bf.setName(field.getValues().get(i).getName()); |
|
128 |
bf.setCount(field.getValues().get(i).getCount() + ""); |
|
129 |
if (map.get(field.getName()) == null) { |
|
130 |
map.put(field.getName(), new ArrayList<String>()); |
|
131 |
} |
|
132 |
|
|
133 |
map.get(field.getName()).add(new Gson().toJson(bf)); |
|
134 |
} |
|
135 |
} |
|
136 |
} |
|
137 |
|
|
138 |
for (Map.Entry<String, List<String>> facetEntry : map.entrySet()) { |
|
139 |
StringBuilder builder = new StringBuilder(); |
|
140 |
builder.append("\"" + facetEntry.getKey() + "\"" + " : "); |
|
141 |
builder.append(facetEntry.getValue()); |
|
142 |
res.add(builder.toString()); |
|
143 |
} |
|
144 |
} |
|
145 |
|
|
146 |
logger.debug("time: " + rsp.getElapsedTime()); |
|
147 |
logger.debug("found: " + docs.getNumFound()); |
|
148 |
logger.debug("docs: " + docs.size()); |
|
149 |
|
|
150 |
for (int i = 0; i < docs.size(); i++) { |
|
151 |
String result = (String) docs.get(i).get("__result"); |
|
152 |
res.add(result); |
|
153 |
} |
|
154 |
|
|
155 |
return res; |
|
156 |
|
|
157 |
} catch (SolrServerException sse) { |
|
158 |
logger.error("Fail to get results from Solr. ", sse); |
|
159 |
|
|
160 |
} catch (IOException ioe) { |
|
161 |
logger.error("Fail to get results from Solr. ", ioe); |
|
162 |
} |
|
163 |
|
|
164 |
return null; |
|
165 |
} |
|
166 |
|
|
167 |
@Override |
|
168 |
public EPR getEpr() { |
|
169 |
return null; |
|
170 |
} |
|
171 |
|
|
172 |
public Map<String,List<String>> newGet(int from, int to, String format, Transformer transformer, |
|
173 |
Transformer oldRefineTransformer, boolean timeout, String timeoutValue) throws SearchServiceException { |
|
174 |
List<String> refineSolrResults = new ArrayList<String>(); |
|
175 |
List<String> searchSolrResults = new ArrayList<String>(); |
|
176 |
|
|
177 |
QueryResponse rsp = null; |
|
178 |
HashMap<String, List<String>> map = new HashMap<String, List<String>>(); |
|
179 |
|
|
180 |
if (from < 0) from = 0; |
|
181 |
if (to < 0) to = 0; |
|
182 |
queryOpts.add("start", from*to + ""); |
|
183 |
queryOpts.add("rows", to +""); |
|
184 |
|
|
185 |
try { |
|
186 |
if (timeout) { |
|
187 |
if (timeoutValue != null && Integer.valueOf(timeoutValue) == -1) { |
|
188 |
logger.warn("Timeout value is -1. Timeout will not be set for query: " + queryOpts.get("q")); |
|
189 |
timeout = false; |
|
190 |
} else if (Integer.valueOf(timeoutValue) < 500) { |
|
191 |
logger.warn("Timeout is less than 500ms. We fall back to the default 1000ms for query:" + queryOpts.get("q")); |
|
192 |
queryOpts.add("timeAllowed", "500"); |
|
193 |
} else if (timeout) { |
|
194 |
queryOpts.add("timeAllowed", timeoutValue); |
|
195 |
} |
|
196 |
} |
|
197 |
} catch (NumberFormatException nfe) { |
|
198 |
logger.warn("NumberFormatException occurred. Timeout not properly set. We fall back to the default 1000ms."); |
|
199 |
queryOpts.add("timeAllowed", "500"); |
|
200 |
} |
|
201 |
|
|
202 |
logger.info("timeout: " + timeout + ", timeout value: " + queryOpts.get("timeAllowed")); |
|
203 |
|
|
204 |
//queryOpts.add("f.resulthostingdatasource.facet.limit", "2"); |
|
205 |
|
|
206 |
long startTime = System.nanoTime(); |
|
207 |
|
|
208 |
try { |
|
209 |
io.micrometer.core.instrument.Timer.Sample timer = Timer.start(registry); |
|
210 |
rsp = solrClient.query(SolrParams.toSolrParams(queryOpts)); |
|
211 |
timer.stop(registry.timer("solr.server.response.duration")); |
|
212 |
|
|
213 |
long estimatedTime = System.nanoTime() - startTime; |
|
214 |
logger.info("Solrj time " + estimatedTime/1000000 + " milliseconds for query:" + queryOpts.get("q") + |
|
215 |
" and facets " + queryOpts.getAll("facet.field") + " and fq " + queryOpts.getAll("fq") + |
|
216 |
" and sorts " + queryOpts.getAll("sort") + " from: " |
|
217 |
+ from + " and size " + to + " with timeout: " + timeout); |
|
218 |
|
|
219 |
/*System.out.println("Solrj time " + estimatedTime/1000000 + " milliseconds for query:" + queryOpts.get("q") + |
|
220 |
" and facets " + queryOpts.getAll("facet.field") + " and fq " + queryOpts.getAll("fq") + |
|
221 |
" and sorts " + queryOpts.getAll("sort") + " from: " |
|
222 |
+ from + " and size " + to); |
|
223 |
*/ |
|
224 |
facetFields = rsp.getFacetFields(); |
|
225 |
|
|
226 |
SolrDocumentList docs = rsp.getResults(); |
|
227 |
|
|
228 |
this.size = docs.getNumFound(); |
|
229 |
|
|
230 |
if (facetFields!=null && !facetFields.isEmpty()) { |
|
231 |
if (format != null && format.equals(MediaType.APPLICATION_JSON)) { |
|
232 |
for (FacetField field : facetFields) { |
|
233 |
map.put(field.getName(), new ArrayList<String>()); |
|
234 |
BrowseField bf = null; |
|
235 |
for (int i = 0; i < field.getValueCount(); i++) { |
|
236 |
bf = new BrowseField(); |
|
237 |
//bf.setId(org.apache.commons.lang3.StringEscapeUtils.escapeJson(field.getValues().get(i).getName())); |
|
238 |
bf.setId(field.getValues().get(i).getName()); |
|
239 |
String[] facetedValues = field.getValues().get(i).getName().split("\\|\\|",2); |
|
240 |
|
|
241 |
|
|
242 |
logger.debug("faceted values " + Arrays.toString(facetedValues)); |
|
243 |
|
|
244 |
if (facetedValues.length > 1) { |
|
245 |
//bf.setName(org.apache.commons.lang3.StringEscapeUtils.escapeJson(facetedValues[1])); |
|
246 |
bf.setName(facetedValues[1]); |
|
247 |
logger.debug("faceted values [1] " + facetedValues[1]); |
|
248 |
|
|
249 |
} else if (field.getValues().get(i).getName().split("_\\:\\:",2).length > 1) { |
|
250 |
//bf.setName(org.apache.commons.lang3.StringEscapeUtils.escapeJson(field.getValues().get(i).getName().split("\\:\\:",2)[1]).replaceAll("\\:\\:", "\\|")); |
|
251 |
bf.setName(field.getValues().get(i).getName().split("\\:\\:",2)[1].replaceAll("\\:\\:", "\\|")); |
|
252 |
|
|
253 |
} else { |
|
254 |
//bf.setName(org.apache.commons.lang3.StringEscapeUtils.escapeJson(field.getValues().get(i).getName())); |
|
255 |
bf.setName(field.getValues().get(i).getName()); |
|
256 |
} |
|
257 |
|
|
258 |
bf.setCount(field.getValues().get(i).getCount() + ""); |
|
259 |
map.get(field.getName()).add(new Gson().toJson(bf)); |
|
260 |
} |
|
261 |
|
|
262 |
} |
|
263 |
|
|
264 |
StringBuilder builder = null; |
|
265 |
|
|
266 |
for (Map.Entry<String, List<String>> facetEntry : map.entrySet()) { |
|
267 |
builder = new StringBuilder(); |
|
268 |
builder.append("\"" + facetEntry.getKey() + "\"" + " : "); |
|
269 |
builder.append(facetEntry.getValue()); |
|
270 |
refineSolrResults.add(builder.toString()); |
|
271 |
} |
Also available in: Unified diff
Creating a new branch for JSON parsing