Revision 53775
Added by Claudio Atzori almost 6 years ago
modules/dnet-index-solr-client/branches/solr75/src/main/java/eu/dnetlib/functionality/index/query/SolrResponseParser.java | ||
---|---|---|
1 |
package eu.dnetlib.functionality.index.query; |
|
2 |
|
|
3 |
import java.io.ByteArrayInputStream; |
|
4 |
import java.io.IOException; |
|
5 |
import java.util.Collection; |
|
6 |
import java.util.List; |
|
7 |
import java.util.Map; |
|
8 |
import java.util.zip.GZIPInputStream; |
|
9 |
|
|
10 |
import com.google.common.base.Predicate; |
|
11 |
import com.google.common.collect.BiMap; |
|
12 |
import com.google.common.collect.Iterables; |
|
13 |
import com.google.common.collect.Lists; |
|
14 |
import eu.dnetlib.data.provision.index.rmi.BrowsingRow; |
|
15 |
import eu.dnetlib.data.provision.index.rmi.GroupResult; |
|
16 |
import eu.dnetlib.functionality.index.utils.IndexFieldUtility; |
|
17 |
import eu.dnetlib.miscutils.functional.UnaryFunction; |
|
18 |
import org.apache.commons.io.IOUtils; |
|
19 |
import org.apache.commons.logging.Log; |
|
20 |
import org.apache.commons.logging.LogFactory; |
|
21 |
import org.apache.solr.client.solrj.response.FacetField; |
|
22 |
import org.apache.solr.client.solrj.response.FacetField.Count; |
|
23 |
import org.apache.solr.client.solrj.response.QueryResponse; |
|
24 |
import org.apache.solr.common.SolrDocument; |
|
25 |
import org.apache.solr.common.SolrDocumentList; |
|
26 |
|
|
27 |
import static eu.dnetlib.miscutils.collections.MappedCollection.listMap; |
|
28 |
|
|
29 |
/** |
|
30 |
* The Class SolrResponseParser. |
|
31 |
*/ |
|
32 |
public class SolrResponseParser extends QueryResponseParser { |
|
33 |
|
|
34 |
/** |
|
35 |
* logger. |
|
36 |
*/ |
|
37 |
private static final Log log = LogFactory.getLog(SolrResponseParser.class); |
|
38 |
|
|
39 |
/** |
|
40 |
* Lower level response. |
|
41 |
*/ |
|
42 |
private QueryResponse queryRsp = null; |
|
43 |
|
|
44 |
/** The wrapper rank. */ |
|
45 |
protected final UnaryFunction<String, SolrDocument> wrapperRank = |
|
46 |
doc -> addRanking( |
|
47 |
getSingleField(doc, IndexFieldUtility.RESULT), |
|
48 |
getSingleField(doc, IndexFieldUtility.SCORE_FIELD)); |
|
49 |
|
|
50 |
/** The wrapper no rank. */ |
|
51 |
protected final UnaryFunction<String, SolrDocument> wrapperNoRank = |
|
52 |
doc -> wrap(getSingleField(doc, IndexFieldUtility.RESULT)); |
|
53 |
|
|
54 |
/** |
|
55 |
* The Constructor. |
|
56 |
* |
|
57 |
* @param highlightUtils |
|
58 |
* the highlight utils |
|
59 |
* @param aliases |
|
60 |
* the aliases |
|
61 |
* @param returnEmptyFields |
|
62 |
* the return empty fields |
|
63 |
* @param includeRanking |
|
64 |
* the include ranking |
|
65 |
* @param response |
|
66 |
* the response |
|
67 |
*/ |
|
68 |
public SolrResponseParser(final UnaryFunction<String, String> highlightUtils, final BiMap<String, String> aliases, final boolean returnEmptyFields, |
|
69 |
final boolean includeRanking, final QueryResponse response) { |
|
70 |
super(highlightUtils, aliases, returnEmptyFields, includeRanking); |
|
71 |
this.queryRsp = response; |
|
72 |
} |
|
73 |
|
|
74 |
/** |
|
75 |
* {@inheritDoc} |
|
76 |
* |
|
77 |
* @see eu.dnetlib.functionality.index.query.QueryResponseParser#getNumFound() |
|
78 |
*/ |
|
79 |
@Override |
|
80 |
public long getNumFound() { |
|
81 |
|
|
82 |
return this.queryRsp.getResults().getNumFound(); |
|
83 |
} |
|
84 |
|
|
85 |
/** |
|
86 |
* {@inheritDoc} |
|
87 |
* |
|
88 |
* @see eu.dnetlib.functionality.index.query.QueryResponseParser#getQueryTime() |
|
89 |
*/ |
|
90 |
@Override |
|
91 |
public int getQueryTime() { |
|
92 |
return queryRsp.getQTime(); |
|
93 |
} |
|
94 |
|
|
95 |
/** |
|
96 |
* {@inheritDoc} |
|
97 |
* |
|
98 |
* @see eu.dnetlib.functionality.index.query.QueryResponseParser#getElapsedTime() |
|
99 |
*/ |
|
100 |
@Override |
|
101 |
public long getElapsedTime() { |
|
102 |
return queryRsp.getElapsedTime(); |
|
103 |
} |
|
104 |
|
|
105 |
/** |
|
106 |
* {@inheritDoc} |
|
107 |
* |
|
108 |
* @see eu.dnetlib.functionality.index.query.QueryResponseParser#getStatus() |
|
109 |
*/ |
|
110 |
@Override |
|
111 |
public String getStatus() { |
|
112 |
return String.valueOf(queryRsp.getStatus()); |
|
113 |
} |
|
114 |
|
|
115 |
/** |
|
116 |
* {@inheritDoc} |
|
117 |
* |
|
118 |
* @see eu.dnetlib.functionality.index.query.QueryResponseParser#getCurrentSize() |
|
119 |
*/ |
|
120 |
@Override |
|
121 |
public int getCurrentSize() { |
|
122 |
return queryRsp.getResults().size(); |
|
123 |
} |
|
124 |
|
|
125 |
/** |
|
126 |
* Gets the query response. |
|
127 |
* |
|
128 |
* @return the query response |
|
129 |
*/ |
|
130 |
public QueryResponse getQueryResponse() { |
|
131 |
return queryRsp; |
|
132 |
} |
|
133 |
|
|
134 |
/** |
|
135 |
* {@inheritDoc} |
|
136 |
* |
|
137 |
* @see eu.dnetlib.functionality.index.query.QueryResponseParser#getResults() |
|
138 |
*/ |
|
139 |
@Override |
|
140 |
public List<String> getResults() { |
|
141 |
return asRankedList(queryRsp.getResults()); |
|
142 |
} |
|
143 |
|
|
144 |
/** |
|
145 |
* {@inheritDoc} |
|
146 |
* |
|
147 |
* @see eu.dnetlib.functionality.index.query.QueryResponseParser#getNumberOfBrowsingResults() |
|
148 |
*/ |
|
149 |
@Override |
|
150 |
public Long getNumberOfBrowsingResults() { |
|
151 |
List<FacetField> ffList = queryRsp.getFacetFields(); |
|
152 |
Long maxCount = 0L; |
|
153 |
|
|
154 |
if (ffList != null) { |
|
155 |
for (FacetField ff : ffList) { |
|
156 |
if (ff != null) { |
|
157 |
Long countFacets = countFacets(ff.getValues()); |
|
158 |
if (countFacets > maxCount) { |
|
159 |
maxCount = countFacets; |
|
160 |
} |
|
161 |
} |
|
162 |
} |
|
163 |
} |
|
164 |
return maxCount; |
|
165 |
} |
|
166 |
|
|
167 |
/** |
|
168 |
* {@inheritDoc} |
|
169 |
* |
|
170 |
* @see eu.dnetlib.functionality.index.query.QueryResponseParser#getBrowsingResults() |
|
171 |
*/ |
|
172 |
@Override |
|
173 |
public List<BrowsingRow> getBrowsingResults() { |
|
174 |
List<BrowsingRow> bresList = Lists.newArrayList(); |
|
175 |
List<GroupResult> facets = Lists.newArrayList(); |
|
176 |
|
|
177 |
final List<FacetField> ffList = queryRsp.getFacetFields(); |
|
178 |
|
|
179 |
Long numberOfBrowsingResults = getNumberOfBrowsingResults(); |
|
180 |
for (int i = 0; (ffList != null) && (i < numberOfBrowsingResults); i++) { |
|
181 |
for (FacetField ff : ffList) { |
|
182 |
|
|
183 |
String name = null; |
|
184 |
if (aliases != null) { |
|
185 |
name = aliases.inverse().get(ff.getName()); |
|
186 |
} |
|
187 |
|
|
188 |
// fix #1456 |
|
189 |
if (name == null) { |
|
190 |
name = ff.getName(); |
|
191 |
} |
|
192 |
|
|
193 |
final Count facet = getFacet(ff, i); |
|
194 |
|
|
195 |
if ((facet != null) && (facet.getCount() > 0)) { |
|
196 |
|
|
197 |
final String value = facet.getName(); |
|
198 |
final int count = (int) facet.getCount(); |
|
199 |
|
|
200 |
if (returnEmptyFields || !value.isEmpty()) { |
|
201 |
facets.add(new GroupResult(name, value, count)); |
|
202 |
} |
|
203 |
} |
|
204 |
} |
|
205 |
|
|
206 |
if (facets.size() > 0) { |
|
207 |
bresList.add(new BrowsingRow(Lists.newArrayList(facets))); |
|
208 |
facets.clear(); |
|
209 |
} |
|
210 |
} |
|
211 |
if (log.isDebugEnabled()) { |
|
212 |
log.debug("BrowsingResult size: " + bresList.size()); |
|
213 |
} |
|
214 |
return bresList; |
|
215 |
} |
|
216 |
|
|
217 |
// /////////////// helpers |
|
218 |
|
|
219 |
/** |
|
220 |
* Gets the facet. |
|
221 |
* |
|
222 |
* @param ff |
|
223 |
* the ff |
|
224 |
* @param pos |
|
225 |
* the pos |
|
226 |
* @return the facet |
|
227 |
*/ |
|
228 |
private Count getFacet(final FacetField ff, final int pos) { |
|
229 |
|
|
230 |
if ((ff.getValues() == null) || (pos >= ff.getValues().size())) return null; |
|
231 |
return ff.getValues().get(pos); |
|
232 |
} |
|
233 |
|
|
234 |
/** |
|
235 |
* Given SolrDocumentList, return a List of Strings, representing it. |
|
236 |
* |
|
237 |
* @param documentList |
|
238 |
* the document list |
|
239 |
* @return the list< string> |
|
240 |
*/ |
|
241 |
private List<String> asRankedList(final SolrDocumentList documentList) { |
|
242 |
|
|
243 |
UnaryFunction<String, SolrDocument> wrapper = includeRanking ? wrapperRank : wrapperNoRank; |
|
244 |
|
|
245 |
if (queryRsp.getHighlighting() != null) return listMap(listMap(documentList, doc -> { |
|
246 |
|
|
247 |
String score = getSingleField(doc, IndexFieldUtility.SCORE_FIELD); |
|
248 |
|
|
249 |
String hl = getHighlighting(getSingleField(doc, IndexFieldUtility.INDEX_RECORD_ID)); |
|
250 |
String res = hl != null ? hl : getSingleField(doc, IndexFieldUtility.RESULT); |
|
251 |
|
|
252 |
return includeRanking ? addRanking(res, score) : wrap(res); |
|
253 |
}), highlightUtils); |
|
254 |
|
|
255 |
return listMap(documentList, wrapper); |
|
256 |
} |
|
257 |
|
|
258 |
/** |
|
259 |
* Converts a String document to |
|
260 |
* |
|
261 |
* <record rank="score"> [document] </record>. |
|
262 |
* |
|
263 |
* @param doc |
|
264 |
* the doc |
|
265 |
* @param score |
|
266 |
* the score |
|
267 |
* @return the string |
|
268 |
*/ |
|
269 |
private String addRanking(final String doc, final String score) { |
|
270 |
return new String("<record rank=\"" + score + "\">" + doc + "</record>"); |
|
271 |
} |
|
272 |
|
|
273 |
/** |
|
274 |
* Wraps the given document as <record> [document] </record>. |
|
275 |
* |
|
276 |
* @param doc |
|
277 |
* the doc |
|
278 |
* @return the string |
|
279 |
*/ |
|
280 |
private String wrap(final String doc) { |
|
281 |
return new String("<record>" + doc + "</record>"); |
|
282 |
} |
|
283 |
|
|
284 |
/** |
|
285 |
* Gets the single field. |
|
286 |
* |
|
287 |
* @param doc |
|
288 |
* the doc |
|
289 |
* @param fieldName |
|
290 |
* the field name |
|
291 |
* @return the single field |
|
292 |
*/ |
|
293 |
@SuppressWarnings("unchecked") |
|
294 |
protected String getSingleField(final SolrDocument doc, final String fieldName) { |
|
295 |
Object value = doc.getFieldValue(fieldName); |
|
296 |
if (value instanceof Collection) return Iterables.getOnlyElement((Iterable<String>) value); |
|
297 |
return String.valueOf(value); |
|
298 |
} |
|
299 |
|
|
300 |
private byte[] base64Decode(final String s) { |
|
301 |
return org.apache.solr.common.util.Base64.base64ToByteArray(s); |
|
302 |
} |
|
303 |
|
|
304 |
private String unzip(final byte[] b) { |
|
305 |
try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(b)) { |
|
306 |
try (GZIPInputStream gzipInputStream = new GZIPInputStream(byteArrayInputStream)) { |
|
307 |
return new String(IOUtils.toByteArray(gzipInputStream)); |
|
308 |
} |
|
309 |
} catch(IOException e) { |
|
310 |
throw new RuntimeException("Failed to unzip content", e); |
|
311 |
} |
|
312 |
} |
|
313 |
|
|
314 |
/** |
|
315 |
* Gets the highlighting. |
|
316 |
* |
|
317 |
* @param docId |
|
318 |
* the doc id |
|
319 |
* @return the highlighting |
|
320 |
*/ |
|
321 |
private String getHighlighting(final String docId) { |
|
322 |
final Map<String, List<String>> highlight = queryRsp.getHighlighting().get(docId); |
|
323 |
|
|
324 |
String result = new String(); |
|
325 |
if ((highlight != null) && (highlight.get(IndexFieldUtility.RESULT) != null)) { |
|
326 |
for (String s : highlight.get(IndexFieldUtility.RESULT)) { |
|
327 |
result = result.concat(s); |
|
328 |
} |
|
329 |
return result; |
|
330 |
} |
|
331 |
return null; |
|
332 |
} |
|
333 |
|
|
334 |
/** |
|
335 |
* helper method. |
|
336 |
* |
|
337 |
* @param facets |
|
338 |
* the list of facets to analyze |
|
339 |
* @return the number of non-empty facets in the list whose count is greater than zero |
|
340 |
*/ |
|
341 |
private Long countFacets(final List<Count> facets) { |
|
342 |
|
|
343 |
if (facets == null) return 0L; |
|
344 |
|
|
345 |
return (long) Iterables.size(Iterables.filter(facets, new Predicate<Count>() { |
|
346 |
|
|
347 |
@Override |
|
348 |
public boolean apply(final Count c) { |
|
349 |
return (c != null) && (c.getName() != null) && !c.getName().isEmpty() && (c.getCount() > 0); |
|
350 |
} |
|
351 |
})); |
|
352 |
} |
|
353 |
|
|
354 |
@Override |
|
355 |
public long getStart() { |
|
356 |
// TODO Auto-generated method stub |
|
357 |
return queryRsp.getResults().getStart(); |
|
358 |
} |
|
359 |
|
|
360 |
} |
modules/dnet-index-solr-client/branches/solr75/src/main/java/eu/dnetlib/functionality/index/query/SolrIndexQueryFactory.java | ||
---|---|---|
1 |
package eu.dnetlib.functionality.index.query; |
|
2 |
|
|
3 |
import java.util.Arrays; |
|
4 |
|
|
5 |
import eu.dnetlib.functionality.cql.lucene.TranslatedQuery; |
|
6 |
import org.apache.commons.logging.Log; |
|
7 |
import org.apache.commons.logging.LogFactory; |
|
8 |
|
|
9 |
import eu.dnetlib.functionality.index.client.AbstractIndexClient; |
|
10 |
import eu.dnetlib.functionality.index.query.Pruner.Result; |
|
11 |
import eu.dnetlib.functionality.index.solr.utils.HighlightUtils; |
|
12 |
import eu.dnetlib.functionality.index.utils.IndexFieldUtility; |
|
13 |
|
|
14 |
/** |
|
15 |
* A factory for creating SolrIndexQuery objects. |
|
16 |
*/ |
|
17 |
public class SolrIndexQueryFactory extends IndexQueryFactory { |
|
18 |
|
|
19 |
/** |
|
20 |
* logger. |
|
21 |
*/ |
|
22 |
private static final Log log = LogFactory.getLog(SolrIndexQueryFactory.class); |
|
23 |
|
|
24 |
/** The Property name SERVICE_HIGHLIGHT_ENABLE. */ |
|
25 |
private static final String SERVICE_HIGHLIGHT_ENABLE = "service.index.solr.highlight.enable"; |
|
26 |
|
|
27 |
/* |
|
28 |
* (non-Javadoc) |
|
29 |
* |
|
30 |
* @see eu.dnetlib.functionality.index.query.IndexQueryFactory#newInstance(eu.dnetlib.functionality.index.cql.TranslatedQuery, |
|
31 |
* eu.dnetlib.functionality.index.query.Pruner.Result, eu.dnetlib.functionality.index.query.QueryLanguage) |
|
32 |
*/ |
|
33 |
@Override |
|
34 |
protected IndexQuery newInstance(final TranslatedQuery cql, final Result res, final QueryLanguage lang) { |
|
35 |
|
|
36 |
switch (lang) { |
|
37 |
case CQL: |
|
38 |
return new SolrIndexQuery(cql, res.getOptionMap()); |
|
39 |
case SOLR: |
|
40 |
return new SolrIndexQuery(res.getNode().toCQL(), res.getOptionMap()); |
|
41 |
default: |
|
42 |
throw new IllegalArgumentException("invalid query language: " + lang); |
|
43 |
} |
|
44 |
} |
|
45 |
|
|
46 |
/* |
|
47 |
* (non-Javadoc) |
|
48 |
* |
|
49 |
* @see eu.dnetlib.functionality.index.query.IndexQueryFactory#setQueryOptions(eu.dnetlib.functionality.index.query.IndexQuery, |
|
50 |
* eu.dnetlib.functionality.index.IndexServerDAO) |
|
51 |
*/ |
|
52 |
@Override |
|
53 |
protected IndexQuery setQueryOptions(final IndexQuery indexQuery, final AbstractIndexClient client) { |
|
54 |
|
|
55 |
final SolrIndexQuery solrQuery = (SolrIndexQuery) indexQuery; |
|
56 |
|
|
57 |
boolean isHighlightEnabled = Boolean.parseBoolean(client.getServiceProperties().get(SERVICE_HIGHLIGHT_ENABLE)); |
|
58 |
if (solrQuery.getHighlight() & isHighlightEnabled) { |
|
59 |
solrQuery.setHighlightFragsize(0).setHighlightSnippets(1).setHighlightSimplePre(HighlightUtils.DEFAULT_HL_PRE) |
|
60 |
.setHighlightSimplePost(HighlightUtils.DEFAULT_HL_POST).addHighlightField(IndexFieldUtility.RESULT) |
|
61 |
.addField(IndexFieldUtility.INDEX_RECORD_ID); |
|
62 |
} |
|
63 |
|
|
64 |
solrQuery.addField(IndexFieldUtility.RESULT); |
|
65 |
if (solrQuery.getFacetFields() != null) { |
|
66 |
log.debug("getFacetFields() " + Arrays.asList(solrQuery.getFacetFields())); |
|
67 |
solrQuery.setFacetMinCount(1); |
|
68 |
} |
|
69 |
|
|
70 |
return solrQuery; |
|
71 |
} |
|
72 |
|
|
73 |
} |
modules/dnet-index-solr-client/branches/solr75/src/main/java/eu/dnetlib/functionality/index/query/SolrIndexQuery.java | ||
---|---|---|
1 |
package eu.dnetlib.functionality.index.query; |
|
2 |
|
|
3 |
import java.util.List; |
|
4 |
import java.util.Map; |
|
5 |
|
|
6 |
import eu.dnetlib.functionality.cql.lucene.QueryOptions; |
|
7 |
import eu.dnetlib.functionality.cql.lucene.TranslatedQuery; |
|
8 |
import eu.dnetlib.functionality.index.utils.IndexFieldUtility; |
|
9 |
import org.apache.commons.logging.Log; |
|
10 |
import org.apache.commons.logging.LogFactory; |
|
11 |
import org.apache.solr.client.solrj.SolrQuery; |
|
12 |
import org.apache.solr.common.params.ModifiableSolrParams; |
|
13 |
import org.apache.solr.common.params.SolrParams; |
|
14 |
|
|
15 |
/** |
|
16 |
* The Class SolrIndexQuery. |
|
17 |
* |
|
18 |
* @author claudio, sandro |
|
19 |
*/ |
|
20 |
public class SolrIndexQuery extends SolrQuery implements IndexQuery { |
|
21 |
|
|
22 |
/** The Constant serialVersionUID. */ |
|
23 |
private static final long serialVersionUID = 1L; |
|
24 |
|
|
25 |
/** |
|
26 |
* logger. |
|
27 |
*/ |
|
28 |
private static final Log log = LogFactory.getLog(SolrIndexQuery.class); |
|
29 |
|
|
30 |
/** |
|
31 |
* Instantiates a new solr index query. |
|
32 |
* |
|
33 |
* @param query |
|
34 |
* the query |
|
35 |
* @param options |
|
36 |
* the options |
|
37 |
*/ |
|
38 |
public SolrIndexQuery(final TranslatedQuery query, final Map<String, List<String>> options) { |
|
39 |
this(query.asLucene(), options); |
|
40 |
|
|
41 |
setCqlParams(query.getOptions()); |
|
42 |
|
|
43 |
log.debug("internal solr query: " + this.toString()); |
|
44 |
} |
|
45 |
|
|
46 |
/** |
|
47 |
* Instantiates a new solr index query. |
|
48 |
* |
|
49 |
* @param query |
|
50 |
* the query |
|
51 |
* @param options |
|
52 |
* the options |
|
53 |
*/ |
|
54 |
public SolrIndexQuery(final String query, final Map<String, List<String>> options) { |
|
55 |
this(query); |
|
56 |
|
|
57 |
// TODO verify that the input options belongs to solr |
|
58 |
super.add(getQueryParams(options)); |
|
59 |
} |
|
60 |
|
|
61 |
/** |
|
62 |
* Instantiates a new solr index query. |
|
63 |
* |
|
64 |
* @param query |
|
65 |
* the query |
|
66 |
*/ |
|
67 |
public SolrIndexQuery(final String query) { |
|
68 |
super(query); |
|
69 |
} |
|
70 |
|
|
71 |
@Override |
|
72 |
public IndexQuery setQueryOffset(final int offset) { |
|
73 |
super.setStart(offset); |
|
74 |
return this; |
|
75 |
} |
|
76 |
|
|
77 |
@Override |
|
78 |
public IndexQuery setQueryLimit(final int limit) { |
|
79 |
super.setRows(limit); |
|
80 |
return this; |
|
81 |
} |
|
82 |
|
|
83 |
@Override |
|
84 |
public IndexQuery setQueryFacetLimit(final int limit) { |
|
85 |
super.setFacetLimit(limit); |
|
86 |
return this; |
|
87 |
} |
|
88 |
|
|
89 |
@Override |
|
90 |
public IndexQuery setQueryFacetOffset(final int offset) { |
|
91 |
super.set("facet.offset", offset); |
|
92 |
return this; |
|
93 |
} |
|
94 |
|
|
95 |
@Override |
|
96 |
public IndexQuery setDSIdFilters(final List<String> dsIds) { |
|
97 |
if (!isAll(dsIds)) { |
|
98 |
for (String id : dsIds) { |
|
99 |
addFilterQuery(IndexFieldUtility.DS_ID + ":\"" + id + "\""); |
|
100 |
} |
|
101 |
} |
|
102 |
return this; |
|
103 |
} |
|
104 |
|
|
105 |
/** |
|
106 |
* Checks if is all. |
|
107 |
* |
|
108 |
* @param dsIds |
|
109 |
* the ds id |
|
110 |
* @return true, if is all |
|
111 |
*/ |
|
112 |
protected boolean isAll(final List<String> dsIds) { |
|
113 |
return (dsIds != null) && (!dsIds.isEmpty()) && (dsIds.size() == 1) && dsIds.get(0).equalsIgnoreCase(IndexFieldUtility.INDEX_DSID_ALL); |
|
114 |
} |
|
115 |
|
|
116 |
/** |
|
117 |
* Convert our option map to a solr option parameter map. |
|
118 |
* |
|
119 |
* @param options |
|
120 |
* input paramter map. |
|
121 |
* @return solr option parameter map. |
|
122 |
*/ |
|
123 |
private SolrParams getQueryParams(final Map<String, List<String>> options) { |
|
124 |
ModifiableSolrParams params = new ModifiableSolrParams(); |
|
125 |
String[] typeTag = new String[] {}; |
|
126 |
|
|
127 |
for (Map.Entry<String, List<String>> entry : options.entrySet()) { |
|
128 |
params.add(entry.getKey(), entry.getValue().toArray(typeTag)); |
|
129 |
} |
|
130 |
return params; |
|
131 |
} |
|
132 |
|
|
133 |
/** |
|
134 |
* Sets the query options. |
|
135 |
* |
|
136 |
* @param options |
|
137 |
* the options. |
|
138 |
*/ |
|
139 |
private void setCqlParams(final QueryOptions options) { |
|
140 |
if (options != null) { |
|
141 |
if (options.getSort() != null) { |
|
142 |
super.addSort(options.getSort().getField(), SolrQuery.ORDER.valueOf(options.getSort().getMode().name())); |
|
143 |
} |
|
144 |
} |
|
145 |
} |
|
146 |
|
|
147 |
} |
modules/dnet-index-solr-client/branches/solr75/src/main/java/eu/dnetlib/functionality/index/query/SolrIndexQueryResponseFactory.java | ||
---|---|---|
1 |
package eu.dnetlib.functionality.index.query; |
|
2 |
|
|
3 |
import org.apache.solr.client.solrj.response.QueryResponse; |
|
4 |
|
|
5 |
import eu.dnetlib.functionality.index.client.IndexClientException; |
|
6 |
import eu.dnetlib.functionality.index.utils.MetadataReference; |
|
7 |
|
|
8 |
/** |
|
9 |
* The Class SolrIndexQueryResponseFactory. |
|
10 |
*/ |
|
11 |
public class SolrIndexQueryResponseFactory extends QueryResponseFactory<QueryResponse> { |
|
12 |
|
|
13 |
/** |
|
14 |
* {@inheritDoc} |
|
15 |
* |
|
16 |
* @throws IndexClientException |
|
17 |
* |
|
18 |
* @see eu.dnetlib.functionality.index.query.QueryResponseFactory#getQueryResponseParser(eu.dnetlib.functionality.index.query.IndexQueryResponse, |
|
19 |
* eu.dnetlib.functionality.index.utils.MetadataReference) |
|
20 |
*/ |
|
21 |
@Override |
|
22 |
public QueryResponseParser getQueryResponseParser(final IndexQueryResponse<QueryResponse> queryRsp, final MetadataReference mdRef) |
|
23 |
throws IndexClientException { |
|
24 |
|
|
25 |
QueryResponse response = queryRsp.getContextualQueryResponse(); |
|
26 |
return new SolrResponseParser(highlightUtils, browseAliases.get(mdRef), returnEmptyFields, includeRanking, response); |
|
27 |
} |
|
28 |
|
|
29 |
} |
modules/dnet-index-solr-client/branches/solr75/src/main/java/eu/dnetlib/functionality/index/query/SolrIndexQueryResponse.java | ||
---|---|---|
1 |
package eu.dnetlib.functionality.index.query; |
|
2 |
|
|
3 |
import org.apache.solr.client.solrj.response.QueryResponse; |
|
4 |
|
|
5 |
/** |
|
6 |
* The Class SolrIndexQueryResponse. |
|
7 |
*/ |
|
8 |
public class SolrIndexQueryResponse implements IndexQueryResponse<QueryResponse> { |
|
9 |
|
|
10 |
private QueryResponse solrQueryResponse; |
|
11 |
|
|
12 |
public SolrIndexQueryResponse(final QueryResponse solrQueryResponse) { |
|
13 |
this.solrQueryResponse = solrQueryResponse; |
|
14 |
} |
|
15 |
|
|
16 |
@Override |
|
17 |
public QueryResponse getContextualQueryResponse() { |
|
18 |
return solrQueryResponse; |
|
19 |
} |
|
20 |
|
|
21 |
} |
modules/dnet-index-solr-client/branches/solr75/src/main/java/eu/dnetlib/functionality/index/client/clients/solr/SolrIndexClient.java | ||
---|---|---|
1 |
package eu.dnetlib.functionality.index.client.clients.solr; |
|
2 |
|
|
3 |
import java.io.IOException; |
|
4 |
import java.util.ArrayList; |
|
5 |
import java.util.HashMap; |
|
6 |
import java.util.List; |
|
7 |
import java.util.Map; |
|
8 |
|
|
9 |
import com.google.common.collect.BiMap; |
|
10 |
import com.google.common.collect.Maps; |
|
11 |
import eu.dnetlib.data.provision.index.rmi.BrowsingRow; |
|
12 |
import eu.dnetlib.data.provision.index.rmi.GroupResult; |
|
13 |
import eu.dnetlib.data.provision.index.rmi.IndexServiceException; |
|
14 |
import eu.dnetlib.functionality.cql.CqlValueTransformerMap; |
|
15 |
import eu.dnetlib.functionality.index.client.AbstractIndexClient; |
|
16 |
import eu.dnetlib.functionality.index.client.IndexClient; |
|
17 |
import eu.dnetlib.functionality.index.client.IndexClientException; |
|
18 |
import eu.dnetlib.functionality.index.client.response.BrowseEntry; |
|
19 |
import eu.dnetlib.functionality.index.client.response.BrowseValueEntry; |
|
20 |
import eu.dnetlib.functionality.index.client.response.LookupResponse; |
|
21 |
import eu.dnetlib.functionality.index.model.Any.ValueType; |
|
22 |
import eu.dnetlib.functionality.index.query.*; |
|
23 |
import eu.dnetlib.functionality.index.solr.cql.SolrTypeBasedCqlValueTransformerMapFactory; |
|
24 |
import eu.dnetlib.functionality.index.utils.MetadataReference; |
|
25 |
import org.apache.commons.logging.Log; |
|
26 |
import org.apache.commons.logging.LogFactory; |
|
27 |
import org.apache.solr.client.solrj.SolrClient; |
|
28 |
import org.apache.solr.client.solrj.SolrServerException; |
|
29 |
import org.apache.solr.client.solrj.impl.CloudSolrClient; |
|
30 |
import org.apache.solr.client.solrj.request.LukeRequest; |
|
31 |
import org.apache.solr.client.solrj.response.LukeResponse; |
|
32 |
import org.apache.solr.client.solrj.response.LukeResponse.FieldInfo; |
|
33 |
import org.apache.solr.client.solrj.response.QueryResponse; |
|
34 |
|
|
35 |
/** |
|
36 |
* The Class SolrIndexClient. |
|
37 |
*/ |
|
38 |
public class SolrIndexClient extends AbstractIndexClient implements IndexClient { |
|
39 |
|
|
40 |
/** The client. */ |
|
41 |
private CloudSolrClient client; |
|
42 |
|
|
43 |
private SolrIndexQueryFactory solrIndexQueryFactory; |
|
44 |
|
|
45 |
/** The query response factory. */ |
|
46 |
private QueryResponseFactory<QueryResponse> queryResponseFactory; |
|
47 |
|
|
48 |
private SolrTypeBasedCqlValueTransformerMapFactory tMapFactory; |
|
49 |
|
|
50 |
private static final Log log = LogFactory.getLog(SolrIndexClient.class); |
|
51 |
|
|
52 |
/** |
|
53 |
* The Constructor. |
|
54 |
* |
|
55 |
* @param format |
|
56 |
* the format |
|
57 |
* @param layout |
|
58 |
* the layout |
|
59 |
* @param interpretation |
|
60 |
* the interpretation |
|
61 |
* @param serviceProperties |
|
62 |
* the service properties |
|
63 |
* @param tMapFactory |
|
64 |
*/ |
|
65 |
public SolrIndexClient(final String format, final String layout, final String interpretation, final Map<String, String> serviceProperties, |
|
66 |
final SolrIndexQueryFactory indexQueryFactory, final QueryResponseFactory<QueryResponse> queryResponseFactory, |
|
67 |
final SolrTypeBasedCqlValueTransformerMapFactory tMapFactory) { |
|
68 |
super(format, layout, interpretation, serviceProperties); |
|
69 |
|
|
70 |
log.debug(String.format("Created a new instance of the index of type %s-%s-%s", format, layout, interpretation)); |
|
71 |
this.solrIndexQueryFactory = indexQueryFactory; |
|
72 |
this.queryResponseFactory = queryResponseFactory; |
|
73 |
this.tMapFactory = tMapFactory; |
|
74 |
|
|
75 |
} |
|
76 |
|
|
77 |
/** |
|
78 |
* Do delete. |
|
79 |
* |
|
80 |
* @param query |
|
81 |
* the CQL query |
|
82 |
* @return true, if do delete |
|
83 |
* @throws IndexServiceException |
|
84 |
* the index service exception |
|
85 |
*/ |
|
86 |
@Override |
|
87 |
public long delete(final String query) throws IndexClientException { |
|
88 |
try { |
|
89 |
log.debug("delete by query: " + query); |
|
90 |
MetadataReference mdRef = new MetadataReference(getFormat(), getLayout(), getInterpretation()); |
|
91 |
SolrIndexQuery translatedQuery = (SolrIndexQuery) solrIndexQueryFactory.getIndexQuery(QueryLanguage.CQL, query, this, mdRef); |
|
92 |
String tquery = translatedQuery.getQuery(); |
|
93 |
translatedQuery.setQueryLimit(0); |
|
94 |
|
|
95 |
SolrIndexQueryResponse rsp = new SolrIndexQueryResponse(client.query(translatedQuery)); |
|
96 |
QueryResponseParser responseParser = queryResponseFactory.getQueryResponseParser(rsp, mdRef); |
|
97 |
long total = responseParser.getNumFound(); |
|
98 |
client.deleteByQuery(tquery); |
|
99 |
client.commit(); |
|
100 |
return total; |
|
101 |
} catch (Exception e) { |
|
102 |
throw new IndexClientException("unable to run delete by query: " + query, e); |
|
103 |
} |
|
104 |
} |
|
105 |
|
|
106 |
/** |
|
107 |
* {@inheritDoc} |
|
108 |
* |
|
109 |
* @throws IndexClientException |
|
110 |
* |
|
111 |
* @see eu.dnetlib.functionality.index.client.IndexClient#browse(String, List, int) |
|
112 |
*/ |
|
113 |
@Override |
|
114 |
public List<BrowseEntry> browse(final String query, final List<String> browseFields, final int max) throws IndexClientException { |
|
115 |
MetadataReference mdRef = new MetadataReference(getFormat(), getLayout(), getInterpretation()); |
|
116 |
SolrIndexQuery translatedQuery = buildBrowseQuery(query, browseFields, max, mdRef); |
|
117 |
return executeBrowseQuery(query, translatedQuery, mdRef, browseFields); |
|
118 |
} |
|
119 |
|
|
120 |
/** |
|
121 |
* {@inheritDoc} |
|
122 |
* |
|
123 |
* @throws IndexClientException |
|
124 |
* |
|
125 |
* @see eu.dnetlib.functionality.index.client.IndexClient#browse(String, List, int, List) |
|
126 |
*/ |
|
127 |
@Override |
|
128 |
public List<BrowseEntry> browse(final String query, final List<String> browseFields, final int max, final List<String> filterQuery) |
|
129 |
throws IndexClientException { |
|
130 |
MetadataReference mdRef = new MetadataReference(getFormat(), getLayout(), getInterpretation()); |
|
131 |
SolrIndexQuery translatedQuery = buildBrowseQuery(query, browseFields, max, mdRef); |
|
132 |
if (filterQuery != null) { |
|
133 |
log.debug("Filter Query:"); |
|
134 |
for (String fq : filterQuery) { |
|
135 |
translatedQuery.addFilterQuery(fq); |
|
136 |
log.debug("- " + fq); |
|
137 |
} |
|
138 |
} |
|
139 |
return executeBrowseQuery(query, translatedQuery, mdRef, browseFields); |
|
140 |
|
|
141 |
} |
|
142 |
|
|
143 |
public SolrIndexQuery buildBrowseQuery(final String query, final List<String> browseFields, final int max, final MetadataReference mdRef) |
|
144 |
throws IndexClientException { |
|
145 |
log.debug("Browse request for the index collection for query:" + query); |
|
146 |
|
|
147 |
SolrIndexQuery translatedQuery = (SolrIndexQuery) solrIndexQueryFactory.getIndexQuery(QueryLanguage.CQL, query, this, mdRef); |
|
148 |
translatedQuery.setFacet(true); |
|
149 |
if (browseFields != null) { |
|
150 |
List<String> browsableFields = solrIndexQueryFactory.getBrowsableFields(browseFields, mdRef); |
|
151 |
log.debug("Browsing fields:"); |
|
152 |
for (String field : browsableFields) { |
|
153 |
translatedQuery.addFacetField(field); |
|
154 |
log.debug("- " + field); |
|
155 |
|
|
156 |
} |
|
157 |
translatedQuery.setFacetLimit(max); |
|
158 |
log.debug("max number of browsing field :" + max); |
|
159 |
} |
|
160 |
return translatedQuery; |
|
161 |
} |
|
162 |
|
|
163 |
private List<BrowseEntry> executeBrowseQuery(final String originalQuery, |
|
164 |
final SolrIndexQuery query, |
|
165 |
final MetadataReference mdRef, |
|
166 |
final List<String> browseFields) throws IndexClientException { |
|
167 |
try { |
|
168 |
SolrIndexQueryResponse response = new SolrIndexQueryResponse(client.query(query)); |
|
169 |
QueryResponseParser responseParser = queryResponseFactory.getQueryResponseParser(response, mdRef); |
|
170 |
List<BrowsingRow> results = responseParser.getBrowsingResults(); |
|
171 |
List<BrowseEntry> out = convertBrowseEntry(browseFields, results, responseParser.getAliases()); |
|
172 |
return out; |
|
173 |
} catch (SolrServerException | IOException e) { |
|
174 |
throw new IndexClientException("Error on executing a query " + originalQuery, e); |
|
175 |
} |
|
176 |
} |
|
177 |
|
|
178 |
/** |
|
179 |
* Creates the connection. |
|
180 |
* |
|
181 |
* @return the string |
|
182 |
* @throws IndexClientException |
|
183 |
* the index client exception |
|
184 |
*/ |
|
185 |
private String getUrl() throws IndexClientException { |
|
186 |
String address = serviceProperties.get(ZK_ADDRESS); |
|
187 |
if (address == null) { throw new IndexClientException("Unable to load a solr client, missing zk address"); } |
|
188 |
return address; |
|
189 |
} |
|
190 |
|
|
191 |
/** |
|
192 |
* Gets the client. |
|
193 |
* |
|
194 |
* @return the client |
|
195 |
* @throws IndexClientException |
|
196 |
* the index client exception |
|
197 |
*/ |
|
198 |
public SolrClient getClient() throws IndexClientException { |
|
199 |
if (this.client == null) { |
|
200 |
String url = getUrl(); |
|
201 |
log.debug("create new Client " + url); |
|
202 |
client = new CloudSolrClient.Builder() |
|
203 |
.withZkHost(url) |
|
204 |
.build(); |
|
205 |
client.setDefaultCollection(String.format("%s-%s-%s", getFormat(), getLayout(), getInterpretation())); |
|
206 |
try { |
|
207 |
client.ping(); |
|
208 |
} catch (Exception e) { |
|
209 |
throw new IndexClientException("oops something went wrong", e); |
|
210 |
} |
|
211 |
} |
|
212 |
return client; |
|
213 |
} |
|
214 |
|
|
215 |
/** |
|
216 |
* Sets the client. |
|
217 |
* |
|
218 |
* @param client |
|
219 |
* the client |
|
220 |
*/ |
|
221 |
public void setClient(final CloudSolrClient client) { |
|
222 |
this.client = client; |
|
223 |
} |
|
224 |
|
|
225 |
@Override |
|
226 |
public LookupResponse lookup(final String query, final List<String> filterQuery, final int from, final int to) throws IndexClientException { |
|
227 |
log.debug("lookup request for the index collection for query:" + query); |
|
228 |
MetadataReference mdRef = new MetadataReference(getFormat(), getLayout(), getInterpretation()); |
|
229 |
SolrIndexQuery translatedQuery = (SolrIndexQuery) solrIndexQueryFactory.getIndexQuery(QueryLanguage.CQL, query, this, mdRef); |
|
230 |
translatedQuery.setQueryOffset(from); |
|
231 |
translatedQuery.setQueryLimit(to - from + 1); |
|
232 |
if (filterQuery != null) { |
|
233 |
for (String fq : filterQuery) { |
|
234 |
translatedQuery.addFilterQuery(fq); |
|
235 |
} |
|
236 |
} |
|
237 |
|
|
238 |
try { |
|
239 |
SolrIndexQueryResponse response = new SolrIndexQueryResponse(client.query(translatedQuery)); |
|
240 |
QueryResponseParser responseParser = queryResponseFactory.getQueryResponseParser(response, mdRef); |
|
241 |
|
|
242 |
return new LookupResponse(responseParser); |
|
243 |
} catch (SolrServerException | IOException e) { |
|
244 |
throw new IndexClientException("Error on executing a query " + query, e); |
|
245 |
} |
|
246 |
|
|
247 |
} |
|
248 |
|
|
249 |
@Override |
|
250 |
public CqlValueTransformerMap getCqlValueTransformerMap(final MetadataReference mdRef) throws IndexClientException { |
|
251 |
try { |
|
252 |
return this.tMapFactory.getIt(readFieldNamesAndTypes(mdRef.toString())); |
|
253 |
} catch (Exception e) { |
|
254 |
throw new IndexClientException(e); |
|
255 |
|
|
256 |
} |
|
257 |
} |
|
258 |
|
|
259 |
@Override |
|
260 |
public IndexQueryFactory getIndexQueryFactory() { |
|
261 |
return solrIndexQueryFactory; |
|
262 |
} |
|
263 |
|
|
264 |
@Override |
|
265 |
public void stop() throws IndexClientException { |
|
266 |
try { |
|
267 |
log.debug("shutdown client: " + serviceProperties.get(ZK_ADDRESS)); |
|
268 |
client.close(); |
|
269 |
} catch (Throwable e) { |
|
270 |
throw new IndexClientException(e); |
|
271 |
} |
|
272 |
} |
|
273 |
|
|
274 |
private List<BrowseEntry> convertBrowseEntry(final List<String> browseFields, final List<BrowsingRow> results, final BiMap<String, String> aliases) { |
|
275 |
|
|
276 |
Map<String, BrowseEntry> mapResult = new HashMap<String, BrowseEntry>(); |
|
277 |
for (BrowsingRow row : results) { |
|
278 |
for (GroupResult groupResult : row.getGroupResult()) { |
|
279 |
String name = groupResult.getName(); |
|
280 |
List<BrowseValueEntry> valuesEntry; |
|
281 |
BrowseEntry entry; |
|
282 |
if (mapResult.containsKey(name)) { |
|
283 |
entry = mapResult.get(name); |
|
284 |
valuesEntry = entry.getValues(); |
|
285 |
if (valuesEntry == null) { |
|
286 |
valuesEntry = new ArrayList<BrowseValueEntry>(); |
|
287 |
entry.setValues(valuesEntry); |
|
288 |
} |
|
289 |
|
|
290 |
} else { |
|
291 |
entry = new BrowseEntry(); |
|
292 |
entry.setField(name); |
|
293 |
entry.setLabel(name); |
|
294 |
valuesEntry = new ArrayList<BrowseValueEntry>(); |
|
295 |
entry.setValues(valuesEntry); |
|
296 |
mapResult.put(name, entry); |
|
297 |
} |
|
298 |
String value = groupResult.getValue(); |
|
299 |
int count = groupResult.getCount(); |
|
300 |
BrowseValueEntry entryValue = new BrowseValueEntry(value, count); |
|
301 |
valuesEntry.add(entryValue); |
|
302 |
} |
|
303 |
} |
|
304 |
List<BrowseEntry> out = new ArrayList<BrowseEntry>(); |
|
305 |
for (String b : browseFields) { |
|
306 |
String inverse = null; |
|
307 |
if (aliases != null) { |
|
308 |
inverse = aliases.get(b) != null ? aliases.get(b) : aliases.inverse().get(b); |
|
309 |
} |
|
310 |
if (mapResult.containsKey(b)) { |
|
311 |
out.add(mapResult.get(b)); |
|
312 |
} else if (mapResult.containsKey(inverse) == true) { |
|
313 |
BrowseEntry data = mapResult.get(inverse); |
|
314 |
data.setField(b); |
|
315 |
out.add(data); |
|
316 |
} |
|
317 |
} |
|
318 |
return out; |
|
319 |
} |
|
320 |
|
|
321 |
private Map<String, ValueType> readFieldNamesAndTypes(final String coreName) throws SolrServerException, IOException, IndexClientException { |
|
322 |
// final SolrServer client = cloudServer.getSolrServer(coreName); |
|
323 |
final LukeRequest request = new LukeRequest(); |
|
324 |
request.setShowSchema(true); |
|
325 |
|
|
326 |
// cloudServer.setDefaultCollection(coreName); |
|
327 |
request.setNumTerms(0); |
|
328 |
final LukeResponse response = request.process(getClient()); |
|
329 |
final Map<String, FieldInfo> fieldInfos = response.getFieldInfo(); |
|
330 |
final Map<String, LukeResponse.FieldTypeInfo> fieldTypeInfos = response.getFieldTypeInfo(); |
|
331 |
final Map<String, ValueType> result = Maps.newHashMap(); |
|
332 |
for (FieldInfo fieldInfo : fieldInfos.values()) { |
|
333 |
LukeResponse.FieldTypeInfo fieldTypeInfo = fieldTypeInfos.get(fieldInfo.getType()); |
|
334 |
final String fieldName = fieldTypeInfo.getName().toLowerCase(); |
|
335 |
final ValueType fieldType = resolveSolrTypeClassName(fieldName); |
|
336 |
result.put(fieldInfo.getName(), fieldType); |
|
337 |
} |
|
338 |
return result; |
|
339 |
} |
|
340 |
|
|
341 |
private ValueType resolveSolrTypeClassName(final String solrTypeName) { |
|
342 |
if (solrTypeName.contains("LongField")) { |
|
343 |
return ValueType.LONG; |
|
344 |
} else if (solrTypeName.contains("IntField")) { |
|
345 |
return ValueType.LONG; |
|
346 |
} else if (solrTypeName.contains("short")) { |
|
347 |
return ValueType.LONG; |
|
348 |
} else if (solrTypeName.contains("float")) { |
|
349 |
return ValueType.DOUBLE; |
|
350 |
} else if (solrTypeName.contains("double")) { |
|
351 |
return ValueType.DOUBLE; |
|
352 |
} else if (solrTypeName.contains("date")) { |
|
353 |
return ValueType.DATETIME; |
|
354 |
} else { |
|
355 |
return ValueType.STRING; |
|
356 |
} |
|
357 |
} |
|
358 |
|
|
359 |
} |
modules/dnet-index-solr-client/branches/solr75/src/main/java/eu/dnetlib/functionality/index/client/clients/solr/SolrIndexClientFactory.java | ||
---|---|---|
1 |
package eu.dnetlib.functionality.index.client.clients.solr; |
|
2 |
|
|
3 |
import java.lang.reflect.Type; |
|
4 |
import java.util.Map; |
|
5 |
|
|
6 |
import org.springframework.beans.factory.annotation.Autowired; |
|
7 |
import org.springframework.beans.factory.annotation.Required; |
|
8 |
|
|
9 |
import com.google.common.reflect.TypeToken; |
|
10 |
import com.google.gson.Gson; |
|
11 |
|
|
12 |
import eu.dnetlib.functionality.index.client.AbstractIndexClientFactory; |
|
13 |
import eu.dnetlib.functionality.index.client.IndexClient; |
|
14 |
import eu.dnetlib.functionality.index.client.IndexClientException; |
|
15 |
import eu.dnetlib.functionality.index.client.IndexClientFactory; |
|
16 |
import eu.dnetlib.functionality.index.query.SolrIndexQueryFactory; |
|
17 |
import eu.dnetlib.functionality.index.query.SolrIndexQueryResponseFactory; |
|
18 |
import eu.dnetlib.functionality.index.solr.cql.SolrTypeBasedCqlValueTransformerMapFactory; |
|
19 |
|
|
20 |
/** |
|
21 |
* The Class SolrIndexClientFactory. |
|
22 |
*/ |
|
23 |
public class SolrIndexClientFactory extends AbstractIndexClientFactory implements IndexClientFactory { |
|
24 |
|
|
25 |
/** The id. */ |
|
26 |
private static String ID = "id"; |
|
27 |
|
|
28 |
/** The configuration. */ |
|
29 |
private String configuration; |
|
30 |
|
|
31 |
/** The type token. */ |
|
32 |
@SuppressWarnings("serial") |
|
33 |
protected Type typeToken = new TypeToken<Map<String, String>>() {}.getType(); |
|
34 |
|
|
35 |
/** The service properties. */ |
|
36 |
private Map<String, String> serviceProperties; |
|
37 |
|
|
38 |
@Autowired |
|
39 |
private SolrIndexQueryFactory solrIndexQueryFactory; |
|
40 |
|
|
41 |
@Autowired |
|
42 |
private SolrIndexQueryResponseFactory solrIndexQueryResponseFactory; |
|
43 |
|
|
44 |
@Autowired |
|
45 |
private SolrTypeBasedCqlValueTransformerMapFactory tMapFactory; |
|
46 |
|
|
47 |
/** |
|
48 |
* Inits the class. |
|
49 |
* |
|
50 |
* @throws IndexClientException |
|
51 |
* the index client exception |
|
52 |
*/ |
|
53 |
@Override |
|
54 |
public void init() throws IndexClientException { |
|
55 |
try { |
|
56 |
serviceProperties = new Gson().fromJson(getConfiguration(), typeToken); |
|
57 |
} catch (Throwable e) { |
|
58 |
throw new IndexClientException("unable to parse configuration: " + getConfiguration(), e); |
|
59 |
} |
|
60 |
} |
|
61 |
|
|
62 |
/** |
|
63 |
* {@inheritDoc} |
|
64 |
* |
|
65 |
* @see eu.dnetlib.functionality.index.client.IndexClientFactory#getBackendId() |
|
66 |
*/ |
|
67 |
@Override |
|
68 |
public String getBackendId() { |
|
69 |
return serviceProperties.get(SolrIndexClientFactory.ID); |
|
70 |
} |
|
71 |
|
|
72 |
/** |
|
73 |
* {@inheritDoc} |
|
74 |
* |
|
75 |
* @throws IndexClientException |
|
76 |
* |
|
77 |
* @see eu.dnetlib.functionality.index.client.IndexClientFactory#getClient(java.lang.String, java.lang.String, java.lang.String) |
|
78 |
*/ |
|
79 |
@Override |
|
80 |
public IndexClient getClient(final String format, final String layout, final String interpretation) throws IndexClientException { |
|
81 |
return new SolrIndexClient(format, layout, interpretation, isQueryTools.getIndexProperties(getBackendId()), solrIndexQueryFactory, |
|
82 |
solrIndexQueryResponseFactory, tMapFactory); |
|
83 |
} |
|
84 |
|
|
85 |
/** |
|
86 |
* Gets the configuration. |
|
87 |
* |
|
88 |
* @return the configuration |
|
89 |
*/ |
|
90 |
public String getConfiguration() { |
|
91 |
return configuration; |
|
92 |
} |
|
93 |
|
|
94 |
/** |
|
95 |
* Sets the configuration. |
|
96 |
* |
|
97 |
* @param configuration |
|
98 |
* the configuration |
|
99 |
*/ |
|
100 |
@Required |
|
101 |
public void setConfiguration(final String configuration) { |
|
102 |
this.configuration = configuration; |
|
103 |
} |
|
104 |
|
|
105 |
} |
modules/dnet-index-solr-client/branches/solr75/src/main/java/eu/dnetlib/functionality/index/solr/cql/SolrTypeBasedCqlValueTransformerMap.java | ||
---|---|---|
1 |
package eu.dnetlib.functionality.index.solr.cql; |
|
2 |
|
|
3 |
import java.util.Map; |
|
4 |
|
|
5 |
import eu.dnetlib.functionality.cql.CqlValueTransformerMap; |
|
6 |
import org.apache.commons.logging.Log; |
|
7 |
import org.apache.commons.logging.LogFactory; |
|
8 |
import org.apache.solr.common.SolrException; |
|
9 |
|
|
10 |
import eu.dnetlib.functionality.index.model.Any.ValueType; |
|
11 |
import eu.dnetlib.miscutils.functional.IdentityFunction; |
|
12 |
import eu.dnetlib.miscutils.functional.UnaryFunction; |
|
13 |
|
|
14 |
/** |
|
15 |
* This class maps the fields in the given index schema with a transformation rule. |
|
16 |
* |
|
17 |
* @author marko |
|
18 |
* |
|
19 |
*/ |
|
20 |
public class SolrTypeBasedCqlValueTransformerMap implements CqlValueTransformerMap { |
|
21 |
|
|
22 |
/** |
|
23 |
* logger. |
|
24 |
*/ |
|
25 |
private static final Log log = LogFactory.getLog(SolrTypeBasedCqlValueTransformerMap.class); // NOPMD by marko on 11/24/08 5:02 PM |
|
26 |
|
|
27 |
/** |
|
28 |
* Index schema. |
|
29 |
*/ |
|
30 |
private final Map<String, ValueType> schema; |
|
31 |
|
|
32 |
/** |
|
33 |
* Map of functions. |
|
34 |
*/ |
|
35 |
private final Map<String, UnaryFunction<String, String>> transformerMap; |
|
36 |
|
|
37 |
/** |
|
38 |
* Create value transformer map bound to a specific schema |
|
39 |
* @param schema |
|
40 |
* @param transformerMap |
|
41 |
*/ |
|
42 |
public SolrTypeBasedCqlValueTransformerMap(final Map<String, ValueType> schema, final Map<String, UnaryFunction<String, String>> transformerMap) { |
|
43 |
this.schema = schema; |
|
44 |
this.transformerMap = transformerMap; |
|
45 |
} |
|
46 |
|
|
47 |
/** |
|
48 |
* {@inheritDoc} |
|
49 |
* |
|
50 |
* @see eu.dnetlib.functionality.cql.CqlValueTransformerMap#transformerFor(java.lang.String) |
|
51 |
*/ |
|
52 |
@Override |
|
53 |
public UnaryFunction<String, String> transformerFor(final String fieldName) { |
|
54 |
try { |
|
55 |
final ValueType field = schema.get(fieldName); |
|
56 |
|
|
57 |
if (field != null) { |
|
58 |
UnaryFunction<String, String> res = transformerMap.get(field.name()); |
|
59 |
if (res != null) { |
|
60 |
return res; |
|
61 |
} |
|
62 |
} |
|
63 |
} catch (SolrException e) { |
|
64 |
log.debug("cannot find field", e); |
|
65 |
} |
|
66 |
return new IdentityFunction<String>(); |
|
67 |
} |
|
68 |
|
|
69 |
} |
modules/dnet-index-solr-client/branches/solr75/src/main/java/eu/dnetlib/functionality/index/solr/cql/SolrTypeBasedCqlValueTransformerMapFactory.java | ||
---|---|---|
1 |
package eu.dnetlib.functionality.index.solr.cql; |
|
2 |
|
|
3 |
import java.util.Map; |
|
4 |
|
|
5 |
import org.springframework.beans.factory.annotation.Required; |
|
6 |
|
|
7 |
import eu.dnetlib.functionality.index.model.Any.ValueType; |
|
8 |
import eu.dnetlib.miscutils.functional.UnaryFunction; |
|
9 |
|
|
10 |
/** |
|
11 |
* Factory for the SolrTypeBasedCqlValueTransformerMap class objects |
|
12 |
* |
|
13 |
* @author claudio |
|
14 |
* |
|
15 |
*/ |
|
16 |
public class SolrTypeBasedCqlValueTransformerMapFactory { |
|
17 |
|
|
18 |
/** |
|
19 |
* Map of functions, injected via spring. |
|
20 |
*/ |
|
21 |
private Map<String, UnaryFunction<String, String>> transformerMap; |
|
22 |
|
|
23 |
/** |
|
24 |
* Method returns a new instance of SolrTypeBasedCqlValueTransformerMap. |
|
25 |
* |
|
26 |
* @param schema |
|
27 |
* @return |
|
28 |
*/ |
|
29 |
public SolrTypeBasedCqlValueTransformerMap getIt(final Map<String, ValueType> schema) { |
|
30 |
return new SolrTypeBasedCqlValueTransformerMap(schema, getTransformerMap()); |
|
31 |
} |
|
32 |
|
|
33 |
@Required |
|
34 |
public void setTransformerMap(Map<String, UnaryFunction<String, String>> transformerMap) { |
|
35 |
this.transformerMap = transformerMap; |
|
36 |
} |
|
37 |
|
|
38 |
public Map<String, UnaryFunction<String, String>> getTransformerMap() { |
|
39 |
return transformerMap; |
|
40 |
} |
|
41 |
|
|
42 |
} |
modules/dnet-index-solr-client/branches/solr75/src/main/java/eu/dnetlib/functionality/index/solr/cql/SimpleDateValueTransformer.java | ||
---|---|---|
1 |
package eu.dnetlib.functionality.index.solr.cql; |
|
2 |
|
|
3 |
import eu.dnetlib.miscutils.functional.UnaryFunction; |
|
4 |
|
|
5 |
/** |
|
6 |
* Simply and not very roboust normalizer for solr dates. Basically it handles well yyyy-mm-dd and |
|
7 |
* yyyy-mm-ddThh:mm:ssZ |
|
8 |
* |
|
9 |
* @author marko |
|
10 |
* |
|
11 |
*/ |
|
12 |
public class SimpleDateValueTransformer implements UnaryFunction<String, String> { |
|
13 |
@Override |
|
14 |
public String evaluate(final String value) { |
|
15 |
if (!value.endsWith("Z")) |
|
16 |
return value + "T00:00:00Z"; |
|
17 |
return value; |
|
18 |
} |
|
19 |
} |
modules/dnet-index-solr-client/branches/solr75/src/main/java/eu/dnetlib/functionality/index/solr/utils/HighlightUtils.java | ||
---|---|---|
1 |
package eu.dnetlib.functionality.index.solr.utils; |
|
2 |
|
|
3 |
import org.apache.oro.text.perl.Perl5Util; |
|
4 |
|
|
5 |
import eu.dnetlib.miscutils.functional.UnaryFunction; |
|
6 |
|
|
7 |
public class HighlightUtils implements UnaryFunction<String, String> { |
|
8 |
|
|
9 |
public final static String DEFAULT_HL_PRE = "[hl]"; |
|
10 |
|
|
11 |
public final static String DEFAULT_HL_POST = "[/hl]"; |
|
12 |
|
|
13 |
private static String CLEAN_HEADER = "s#\\[/?hl\\]##gm"; |
|
14 |
private static String CLEAN_REGEX_OPEN = "<([^>]*)\\[hl\\]([^>]*)>"; |
|
15 |
private static String CLEAN_REGEX_CLOSE = "<([^>]*)\\[\\/hl\\]([^>]*)>"; |
|
16 |
|
|
17 |
// private static String CLEAN_REGEX_OPEN = "s#<([^>]*)\\[hl\\]([^>]*)>#<$1$2>#gm"; |
|
18 |
// private static String CLEAN_REGEX_CLOSE = "s#<([^>]*)\\[\\/hl\\]([^>]*)>#<$1$2>#gm"; |
|
19 |
|
|
20 |
private Perl5Util p5util = new Perl5Util(); |
|
21 |
|
|
22 |
@Override |
|
23 |
public String evaluate(final String doc) { |
|
24 |
String[] chunk = doc.split("</header>"); |
|
25 |
String string = cleanHeader(chunk[0]) + "</header>" + cleanBody(chunk[1]); |
|
26 |
return string; |
|
27 |
} |
|
28 |
|
|
29 |
private String cleanHeader(final String header) { |
|
30 |
return p5util.substitute(CLEAN_HEADER, header); |
|
31 |
} |
|
32 |
|
|
33 |
// TODO: implement a faster way to do this |
|
34 |
private String cleanBody(final String body) { |
|
35 |
String res = body.replaceAll(CLEAN_REGEX_OPEN, "<$1$2>").replaceAll(CLEAN_REGEX_CLOSE, "<$1$2>"); |
|
36 |
|
|
37 |
if (res.equals(body)) return res; |
|
38 |
|
|
39 |
return cleanBody(res); |
|
40 |
} |
|
41 |
|
|
42 |
} |
modules/dnet-index-solr-client/branches/solr75/src/main/java/eu/dnetlib/functionality/index/model/util/SolrIndexDocument.java | ||
---|---|---|
1 |
package eu.dnetlib.functionality.index.model.util; |
|
2 |
|
|
3 |
import java.util.Collection; |
|
4 |
import java.util.HashMap; |
|
5 |
import java.util.Map; |
|
6 |
|
|
7 |
import org.apache.solr.client.solrj.response.SpellCheckResponse.Collation; |
|
8 |
import org.apache.solr.common.SolrInputDocument; |
|
9 |
import org.apache.solr.common.SolrInputField; |
|
10 |
|
|
11 |
import eu.dnetlib.functionality.index.model.Any.ValueType; |
|
12 |
import eu.dnetlib.functionality.index.model.document.AbstractIndexDocument; |
|
13 |
|
|
14 |
// TODO: Auto-generated Javadoc |
|
15 |
/** |
|
16 |
* The Class SolrIndexDocument an implementation of the index document for SOLR. |
|
17 |
*/ |
|
18 |
public class SolrIndexDocument extends AbstractIndexDocument { |
|
19 |
|
|
20 |
/** |
|
21 |
* Instantiates a new solr index document. |
|
22 |
* |
|
23 |
* @param schema |
|
24 |
* the schema |
|
25 |
* @param dsId |
|
26 |
* the ds id |
|
27 |
*/ |
|
28 |
public SolrIndexDocument(final Map<String, ValueType> schema, final String dsId) { |
|
29 |
super(schema, dsId); |
|
30 |
} |
|
31 |
|
|
32 |
/** |
|
33 |
* Instantiates a new solr index document. |
|
34 |
* |
|
35 |
* @param schema |
|
36 |
* the schema |
|
37 |
* @param dsId |
|
38 |
* the ds id |
|
39 |
* @param solrDocument |
|
40 |
* the solr document |
|
41 |
*/ |
|
42 |
public SolrIndexDocument(final Map<String, ValueType> schema, final String dsId, final SolrInputDocument solrDocument) { |
|
43 |
super(schema, dsId); |
|
44 |
addFields(solrDocument); |
|
45 |
} |
|
46 |
|
|
47 |
/** |
|
48 |
* Adds the fields. |
|
49 |
* |
|
50 |
* @param solrDocument |
|
51 |
* the solr document |
|
52 |
*/ |
|
53 |
private void addFields(final SolrInputDocument solrDocument) { |
|
54 |
for (String name : solrDocument.getFieldNames()) { |
|
55 |
Collection<Object> fieldValues = solrDocument.getFieldValues(name); |
|
56 |
if(fieldValues.size()>1) |
|
57 |
addField(name, fieldValues); |
|
58 |
else if (fieldValues.size()==1) |
|
59 |
addField(name, fieldValues.iterator().next()); |
|
60 |
} |
|
61 |
} |
|
62 |
|
|
63 |
/** |
|
64 |
* Sets the content. |
|
65 |
* |
|
66 |
* @param solrDocument |
|
67 |
* the new content |
|
68 |
*/ |
|
69 |
public void setContent(final SolrInputDocument solrDocument) { |
|
70 |
addFields(solrDocument); |
|
71 |
} |
|
72 |
|
|
73 |
/** |
|
74 |
* Gets the solr document. |
|
75 |
* |
|
76 |
* @return the solr document |
|
77 |
*/ |
|
78 |
public SolrInputDocument getSolrDocument() { |
|
79 |
|
|
80 |
Map<String, SolrInputField> data = new HashMap<String, SolrInputField>(); |
|
81 |
for (String key : fields.keySet()) { |
|
82 |
SolrInputField solrField = new SolrInputField(key); |
|
83 |
for (Object o : fields.get(key)) { |
|
84 |
solrField.addValue(o, 1.0f); |
|
85 |
} |
|
86 |
data.put(key, solrField); |
|
87 |
} |
|
88 |
return new SolrInputDocument(data); |
|
89 |
} |
|
90 |
} |
modules/dnet-index-solr-client/branches/solr75/src/main/java/eu/dnetlib/functionality/index/model/util/AnySolrUtil.java | ||
---|---|---|
1 |
package eu.dnetlib.functionality.index.model.util; |
|
2 |
|
|
3 |
import java.util.Iterator; |
|
4 |
import java.util.Map.Entry; |
|
5 |
|
|
6 |
import org.apache.solr.common.SolrDocumentList; |
|
7 |
import org.apache.solr.common.util.NamedList; |
|
8 |
|
|
9 |
import eu.dnetlib.functionality.index.model.AnyMap; |
|
10 |
import eu.dnetlib.functionality.index.model.DataFactory; |
|
11 |
import eu.dnetlib.functionality.index.model.InvalidValueTypeException; |
|
12 |
import eu.dnetlib.functionality.index.model.Value; |
|
13 |
import eu.dnetlib.functionality.index.model.impl.DefaultDataFactoryImpl; |
|
14 |
|
|
15 |
/** |
|
16 |
* The Class AnySolrUtil. |
|
17 |
*/ |
|
18 |
public class AnySolrUtil extends AnyUtil { |
|
19 |
|
|
20 |
/** |
|
21 |
* Convert named list to any map. |
|
22 |
* |
|
23 |
* @param list |
|
24 |
* the list |
|
25 |
* @param map |
|
26 |
* the map |
|
27 |
* @return the any map |
|
28 |
*/ |
|
29 |
@SuppressWarnings("unchecked") |
|
30 |
public static AnyMap convertNamedListToAnyMap(final NamedList<Object> list, final AnyMap map) { |
|
31 |
final Iterator<Entry<String, Object>> it = list.iterator(); |
|
32 |
while (it.hasNext()) { |
|
33 |
Entry<String, Object> entry = it.next(); |
|
34 |
final String key = entry.getKey(); |
|
35 |
final Object obj = entry.getValue(); |
|
36 |
if (obj instanceof NamedList<?>) { |
|
37 |
final AnyMap subMap = map.getMap(key, true); |
|
38 |
convertNamedListToAnyMap((NamedList<Object>) obj, subMap); |
|
39 |
} else if (obj instanceof SolrDocumentList) { |
|
40 |
SolrDocumentList docList = (SolrDocumentList) obj; |
|
41 |
AnyMap response = DataFactory.DEFAULT.createAnyMap(); |
|
42 |
response.put("numFound", docList.getNumFound()); |
|
43 |
response.put("start", docList.getStart()); |
|
44 |
response.put("maxScore", docList.getMaxScore()); |
|
45 |
response.put("docs", objectToAny(obj)); |
|
46 |
map.put("response", response); |
|
47 |
} else { |
|
48 |
try { |
|
49 |
final Value value = DataFactory.DEFAULT.autoConvertValue(obj); |
|
50 |
map.put(key, value); |
|
51 |
} catch (InvalidValueTypeException exception) { |
|
52 |
; // skip |
|
53 |
} |
|
54 |
} |
|
55 |
} |
|
56 |
return map; |
|
57 |
} |
|
58 |
|
|
59 |
/** |
|
60 |
* Convert named list to any map. |
|
61 |
* |
|
62 |
* @param list |
|
63 |
* the list |
|
64 |
* @return the any map |
|
65 |
*/ |
|
66 |
public static AnyMap convertNamedListToAnyMap(final NamedList<Object> list) { |
|
67 |
return convertNamedListToAnyMap(list, DefaultDataFactoryImpl.INSTANCE.createAnyMap()); |
|
68 |
} |
|
69 |
|
|
70 |
} |
modules/dnet-index-solr-client/branches/solr75/src/main/resources/eu/dnetlib/functionality/index/solr/applicationContext-solr-index-common.xml | ||
---|---|---|
1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
2 |
<beans xmlns="http://www.springframework.org/schema/beans" |
|
3 |
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws" |
|
4 |
xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing" |
Also available in: Unified diff
branch for solr 7.5.0