Revision 44602
Added by Katerina Iatropoulou over 7 years ago
| SolrResultSet.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.data.search.solr; |
| 2 | 2 |
|
| 3 | 3 |
import com.google.gson.Gson; |
| 4 |
import eu.dnetlib.data.search.utils.SolrResultSetOptionsUtil; |
|
| 5 |
import eu.dnetlib.data.search.utils.SolrResultsFormatter; |
|
| 4 | 6 |
import eu.dnetlib.domain.EPR; |
| 5 |
import eu.dnetlib.functionality.index.cql.CqlTranslator; |
|
| 6 | 7 |
import eu.dnetlib.functionality.index.cql.CqlTranslatorImpl; |
| 7 | 8 |
import eu.dnetlib.functionality.index.cql.TranslatedQuery; |
| 8 | 9 |
import gr.uoa.di.driver.enabling.resultset.ResultSet; |
| 10 |
import org.apache.commons.lang.StringEscapeUtils; |
|
| 9 | 11 |
import org.apache.log4j.Logger; |
| 10 | 12 |
import org.apache.solr.client.solrj.SolrServerException; |
| 11 | 13 |
import org.apache.solr.client.solrj.impl.CloudSolrServer; |
| ... | ... | |
| 18 | 20 |
|
| 19 | 21 |
import java.io.IOException; |
| 20 | 22 |
import java.util.ArrayList; |
| 21 |
import java.util.Collections; |
|
| 22 | 23 |
import java.util.HashMap; |
| 23 | 24 |
import java.util.List; |
| 25 |
import java.util.Map; |
|
| 24 | 26 |
|
| 25 | 27 |
/** |
| 26 | 28 |
* Created by antleb on 2/4/14. |
| ... | ... | |
| 33 | 35 |
private CloudSolrServer solrClient = null; |
| 34 | 36 |
|
| 35 | 37 |
private NamedList<String> queryOpts = new NamedList<String>(); |
| 36 |
int size = -1;
|
|
| 38 |
long size = -1;
|
|
| 37 | 39 |
|
| 38 | 40 |
|
| 39 | 41 |
public SolrResultSet(EPR epr, CloudSolrServer solrClient) throws IOException, CQLParseException {
|
| 40 | 42 |
this.epr = epr; |
| 41 | 43 |
this.solrClient = solrClient; |
| 44 |
this.queryOpts = SolrResultSetOptionsUtil.extractQueryOptions(epr.getParameter("query"));
|
|
| 42 | 45 |
|
| 43 |
CqlTranslator translator = new CqlTranslatorImpl(); |
|
| 44 |
String cqlQuery = epr.getParameter("query");
|
|
| 45 |
//logger.debug("epr query param: " + cqlQuery);
|
|
| 46 |
String[] queryParts = cqlQuery.split("&groupby=");
|
|
| 47 |
|
|
| 48 |
//logger.debug("Got query " + cqlQuery);
|
|
| 49 |
//for (int i = 0; i < queryParts.length; i++) {
|
|
| 50 |
// logger.debug("queryParts [" + i + "] = '" + queryParts[i] +"'");
|
|
| 51 |
//} |
|
| 52 |
|
|
| 53 |
TranslatedQuery translatedQuery = translator.getTranslatedQuery(queryParts[0].replace("query=",""));
|
|
| 54 |
|
|
| 55 |
if (epr.getParameter("action").equals("lookup")) {
|
|
| 56 |
queryOpts.add("q", translatedQuery.asLucene());
|
|
| 57 |
queryOpts.add("fl", "__result");
|
|
| 58 |
|
|
| 59 |
if (translatedQuery.getOptions() != null && translatedQuery.getOptions().getSort()!= null ) {
|
|
| 60 |
queryOpts.add("sort", translatedQuery.getOptions().getSort().getField() + " " + translatedQuery.getOptions().getSort().getMode());
|
|
| 61 |
} |
|
| 62 |
|
|
| 63 |
} else if (epr.getParameter("action").equals("browse")) {
|
|
| 64 |
logger.debug("Browse query: " + translatedQuery.asLucene());
|
|
| 65 |
|
|
| 66 |
queryOpts.add("q", translatedQuery.asLucene());
|
|
| 67 |
queryOpts.add("facet", "true");
|
|
| 68 |
queryOpts.add("facet.mincount", "1");
|
|
| 69 |
|
|
| 70 |
if (queryParts.length > 1) {
|
|
| 71 |
|
|
| 72 |
queryOpts.add("facet.threads", queryParts[1].split(",").length + "");
|
|
| 73 |
|
|
| 74 |
for (String field:queryParts[1].split(","))
|
|
| 75 |
queryOpts.add("facet.field", field);
|
|
| 76 |
} |
|
| 77 |
} |
|
| 78 |
|
|
| 79 |
|
|
| 80 |
queryOpts.add("shards.tolerant","true");
|
|
| 81 |
|
|
| 82 | 46 |
String layout = epr.getParameter("layout");
|
| 83 | 47 |
String mdformat = epr.getParameter("mdformat");
|
| 84 | 48 |
String interpretation = epr.getParameter("interpretation");
|
| ... | ... | |
| 86 | 50 |
solrClient.setDefaultCollection(mdformat + "-" + layout + "-" + interpretation); |
| 87 | 51 |
} |
| 88 | 52 |
|
| 53 |
|
|
| 54 |
|
|
| 89 | 55 |
@Override |
| 90 | 56 |
public boolean isOpen() {
|
| 91 | 57 |
return true; |
| ... | ... | |
| 102 | 68 |
} |
| 103 | 69 |
|
| 104 | 70 |
@Override |
| 71 |
@Deprecated |
|
| 105 | 72 |
public int size() {
|
| 106 | 73 |
if (size == -1) {
|
| 107 | 74 |
try {
|
| ... | ... | |
| 111 | 78 |
} |
| 112 | 79 |
} |
| 113 | 80 |
|
| 114 |
return size; |
|
| 81 |
return (int)size;
|
|
| 115 | 82 |
} |
| 116 | 83 |
|
| 84 |
@Deprecated |
|
| 117 | 85 |
private int getSize() throws SolrServerException {
|
| 118 | 86 |
//logger.debug("Query opts" + queryOpts);
|
| 119 | 87 |
QueryResponse rsp = null; |
| ... | ... | |
| 140 | 108 |
return 0; |
| 141 | 109 |
} |
| 142 | 110 |
|
| 111 |
|
|
| 143 | 112 |
@Override |
| 113 |
@Deprecated |
|
| 144 | 114 |
public List<String> getElements(int from, int to) {
|
| 145 | 115 |
return get(from, to); |
| 146 | 116 |
} |
| 147 | 117 |
|
| 118 |
List<FacetField> facetFields = null; |
|
| 119 |
|
|
| 148 | 120 |
@Override |
| 121 |
@Deprecated |
|
| 149 | 122 |
public List<String> get(int from, int to) {
|
| 123 |
List<String> res = new ArrayList<String>(); |
|
| 150 | 124 |
|
| 151 |
logger.debug("Getting records from " + from + " to " + to);
|
|
| 125 |
QueryResponse rsp = null;
|
|
| 152 | 126 |
|
| 153 |
if ("lookup".equals(epr.getParameter("action")))
|
|
| 154 |
return getDocumentResults(from, to); |
|
| 127 |
HashMap<String, List<String>> map = new HashMap<String, List<String>>(); |
|
| 155 | 128 |
|
| 156 |
else if ("browse".equals(epr.getParameter("action"))) {
|
|
| 157 |
return getBrowseResults(from, to); |
|
| 158 |
} |
|
| 129 |
logger.debug("from: " + from);
|
|
| 130 |
logger.debug("to: " + to);
|
|
| 159 | 131 |
|
| 160 |
return null; |
|
| 161 |
} |
|
| 162 | 132 |
|
| 163 |
List<FacetField> facetFields = null; |
|
| 164 |
/* |
|
| 165 |
private List<String> getBrowseResults(int from, int to) {
|
|
| 166 |
List<String> res = new ArrayList<String>(); |
|
| 133 |
queryOpts.add("start", from+1 + "");
|
|
| 134 |
queryOpts.add("rows", to + 1+"");
|
|
| 167 | 135 |
|
| 136 |
try {
|
|
| 137 |
rsp = solrClient.query(SolrParams.toSolrParams(queryOpts)); |
|
| 138 |
facetFields = rsp.getFacetFields(); |
|
| 139 |
SolrDocumentList docs = rsp.getResults(); |
|
| 168 | 140 |
|
| 169 |
if (facetFields == null) {
|
|
| 170 |
try {
|
|
| 171 |
QueryResponse rsp = null; |
|
| 141 |
if (facetFields!=null && !facetFields.isEmpty()) {
|
|
| 142 |
for (int i = from - 1; i < to; i++) {
|
|
| 143 |
for (FacetField field : facetFields) {
|
|
| 144 |
if (field.getValueCount() > i) {
|
|
| 145 |
BrowseField bf = new BrowseField(); |
|
| 146 |
bf.setId(field.getValues().get(i).getName()); |
|
| 147 |
bf.setName(field.getValues().get(i).getName()); |
|
| 148 |
bf.setCount(field.getValues().get(i).getCount() + ""); |
|
| 149 |
if (map.get(field.getName()) == null) {
|
|
| 150 |
map.put(field.getName(), new ArrayList<String>()); |
|
| 151 |
} |
|
| 172 | 152 |
|
| 173 |
synchronized (solrClient) {
|
|
| 174 |
rsp = solrClient.query(SolrParams.toSolrParams(queryOpts)); |
|
| 153 |
map.get(field.getName()).add(new Gson().toJson(bf)); |
|
| 154 |
} |
|
| 155 |
} |
|
| 175 | 156 |
} |
| 176 | 157 |
|
| 177 |
facetFields = rsp.getFacetFields(); |
|
| 178 |
} catch (SolrServerException e) {
|
|
| 179 |
e.printStackTrace(); |
|
| 158 |
for (Map.Entry<String, List<String>> facetEntry : map.entrySet()) {
|
|
| 159 |
StringBuilder builder = new StringBuilder(); |
|
| 160 |
builder.append("\"" + facetEntry.getKey() + "\"" + " : ");
|
|
| 161 |
builder.append(facetEntry.getValue()); |
|
| 162 |
res.add(builder.toString()); |
|
| 163 |
} |
|
| 180 | 164 |
} |
| 181 | 165 |
|
| 166 |
logger.debug("time: " + rsp.getElapsedTime());
|
|
| 167 |
logger.debug("found: " + docs.getNumFound());
|
|
| 168 |
logger.debug("docs: " + docs.size());
|
|
| 182 | 169 |
|
| 183 |
for (int i = from - 1; i < to; i++) {
|
|
| 184 |
StringBuilder sb = new StringBuilder(); |
|
| 170 |
for (int i = 0; i < docs.size(); i++) {
|
|
| 171 |
String result = ((ArrayList<String>) docs.get(i).get("__result")).get(0);
|
|
| 172 |
res.add(result); |
|
| 173 |
} |
|
| 185 | 174 |
|
| 186 |
sb.append("<row>");
|
|
| 175 |
return res;
|
|
| 187 | 176 |
|
| 188 |
for (FacetField field:facetFields) {
|
|
| 189 |
if (field.getValueCount() > i) {
|
|
| 190 |
sb.append("<groupresult field=\"").append(field.getName()).append("\">");
|
|
| 191 |
sb.append("<count>").append(field.getValues().get(i).getCount()).append("</count>");
|
|
| 192 |
sb.append("<value>").append(StringEscapeUtils.escapeXml(field.getValues().get(i).getName())).append("</value>");
|
|
| 193 |
sb.append("</groupresult>");
|
|
| 194 |
} |
|
| 195 |
} |
|
| 196 |
|
|
| 197 |
sb.append("</row>");
|
|
| 198 |
|
|
| 199 |
// logger.debug("row: " + sb.toString());
|
|
| 200 |
|
|
| 201 |
res.add(sb.toString()); |
|
| 202 |
} |
|
| 177 |
} catch (SolrServerException sse) {
|
|
| 178 |
logger.error(sse); |
|
| 203 | 179 |
} |
| 204 | 180 |
|
| 205 |
return res;
|
|
| 206 |
}*/
|
|
| 181 |
return null;
|
|
| 182 |
} |
|
| 207 | 183 |
|
| 208 |
private List<String> getBrowseResults (int from, int to) {
|
|
| 209 |
List<String> res = new ArrayList<String>(); |
|
| 184 |
public Map<String,List<String>> newGet(int from, int to, String format) {
|
|
| 185 |
List<String> refineSolrResults = new ArrayList<String>(); |
|
| 186 |
List<String> searchSolrResults = new ArrayList<String>(); |
|
| 210 | 187 |
|
| 188 |
QueryResponse rsp = null; |
|
| 211 | 189 |
HashMap<String, List<String>> map = new HashMap<String, List<String>>(); |
| 212 |
if (facetFields == null) {
|
|
| 213 |
try {
|
|
| 214 |
QueryResponse rsp = null; |
|
| 215 | 190 |
|
| 216 |
synchronized (solrClient) {
|
|
| 217 |
rsp = solrClient.query(SolrParams.toSolrParams(queryOpts)); |
|
| 218 |
} |
|
| 191 |
logger.debug("from: " + from);
|
|
| 192 |
logger.debug("to: " + to);
|
|
| 219 | 193 |
|
| 220 |
facetFields = rsp.getFacetFields(); |
|
| 221 |
} catch (SolrServerException e) {
|
|
| 222 |
e.printStackTrace(); |
|
| 223 |
} |
|
| 194 |
queryOpts.add("start", from+1 + "");
|
|
| 195 |
queryOpts.add("rows", to +"");
|
|
| 224 | 196 |
|
| 225 |
logger.debug("HELLO from " + from + " to " + to);
|
|
| 197 |
try {
|
|
| 198 |
rsp = solrClient.query(SolrParams.toSolrParams(queryOpts)); |
|
| 199 |
facetFields = rsp.getFacetFields(); |
|
| 200 |
|
|
| 201 |
SolrDocumentList docs = rsp.getResults(); |
|
| 226 | 202 |
|
| 227 |
for (int i = from - 1; i < to; i++) {
|
|
| 228 |
for (FacetField field : facetFields) {
|
|
| 229 |
if (field.getValueCount() > i) {
|
|
| 230 |
BrowseField bf = new BrowseField(); |
|
| 231 |
bf.setId(field.getValues().get(i).getName()); |
|
| 232 |
bf.setName(field.getValues().get(i).getName()); |
|
| 233 |
bf.setCount(field.getValues().get(i).getCount() + ""); |
|
| 234 |
if (map.get(field.getName()) == null) {
|
|
| 203 |
this.size = docs.getNumFound(); |
|
| 204 |
|
|
| 205 |
if (facetFields!=null && !facetFields.isEmpty()) {
|
|
| 206 |
if (format != null && format.equalsIgnoreCase("json")) {
|
|
| 207 |
for (FacetField field : facetFields) {
|
|
| 235 | 208 |
map.put(field.getName(), new ArrayList<String>()); |
| 209 |
BrowseField bf = null; |
|
| 210 |
logger.debug("field " + field.getValues());
|
|
| 211 |
logger.debug("field value " + field.getValueCount());
|
|
| 212 |
for (int i = 0; i < field.getValueCount(); i++) {
|
|
| 213 |
bf = new BrowseField(); |
|
| 214 |
bf.setId(field.getValues().get(i).getName()); |
|
| 215 |
bf.setName(field.getValues().get(i).getName()); |
|
| 216 |
bf.setCount(field.getValues().get(i).getCount() + ""); |
|
| 217 |
map.get(field.getName()).add(new Gson().toJson(bf)); |
|
| 236 | 218 |
} |
| 237 | 219 |
|
| 238 |
map.get(field.getName()).add(new Gson().toJson(bf)); |
|
| 239 | 220 |
} |
| 240 |
} |
|
| 241 |
} |
|
| 242 | 221 |
|
| 243 |
for (String index : map.keySet()) {
|
|
| 244 |
StringBuilder builder = new StringBuilder(); |
|
| 245 |
builder.append("\"" + index + "\"" + " : ");
|
|
| 246 |
builder.append(map.get(index)); |
|
| 247 |
res.add(builder.toString()); |
|
| 248 |
} |
|
| 249 |
} |
|
| 222 |
StringBuilder builder = null; |
|
| 250 | 223 |
|
| 251 |
return res; |
|
| 252 |
} |
|
| 224 |
for (Map.Entry<String, List<String>> facetEntry : map.entrySet()) {
|
|
| 225 |
builder = new StringBuilder(); |
|
| 226 |
builder.append("\"" + facetEntry.getKey() + "\"" + " : ");
|
|
| 227 |
builder.append(facetEntry.getValue()); |
|
| 228 |
refineSolrResults.add(builder.toString()); |
|
| 229 |
} |
|
| 253 | 230 |
|
| 254 |
private List<String> getDocumentResults(int from, int to) {
|
|
| 255 |
try {
|
|
| 256 |
QueryResponse rsp = null; |
|
| 257 |
NamedList<String> extraOpts = new NamedList<String>(); |
|
| 231 |
} else { //the old implementation & xml as default
|
|
| 232 |
for (int i = from - 1; i < facetFields.size(); i++) {
|
|
| 233 |
StringBuilder sb = new StringBuilder(); |
|
| 234 |
sb.append("<row>");
|
|
| 235 |
for (FacetField field : facetFields) {
|
|
| 236 |
if (field.getValueCount() > i) {
|
|
| 237 |
sb.append("<groupresult field=\"").append(field.getName()).append("\">");
|
|
| 238 |
sb.append("<count>").append(field.getValues().get(i).getCount()).append("</count>");
|
|
| 239 |
sb.append("<value>").append(StringEscapeUtils.escapeXml(field.getValues().get(i).getName())).append("</value>");
|
|
| 240 |
sb.append("</groupresult>");
|
|
| 241 |
} |
|
| 242 |
} |
|
| 243 |
sb.append("</row>");
|
|
| 244 |
refineSolrResults.add(sb.toString()); |
|
| 245 |
} |
|
| 246 |
} |
|
| 247 |
} |
|
| 258 | 248 |
|
| 259 |
extraOpts.add("start", (from - 1) + "");
|
|
| 260 |
extraOpts.add("rows", (to - from) + 1 + "");
|
|
| 261 |
extraOpts.addAll(queryOpts); |
|
| 262 |
|
|
| 263 |
synchronized (solrClient) {
|
|
| 264 |
rsp = solrClient.query(SolrParams.toSolrParams(extraOpts)); |
|
| 249 |
for (int i = 0; i < docs.size(); i++) {
|
|
| 250 |
String result = ((ArrayList<String>) docs.get(i).get("__result")).get(0);
|
|
| 251 |
if (format != null && format.equalsIgnoreCase("json")) {
|
|
| 252 |
searchSolrResults.add(SolrResultsFormatter.xml2Json(result)); |
|
| 253 |
} else { // default xml
|
|
| 254 |
searchSolrResults.add(result); |
|
| 255 |
} |
|
| 265 | 256 |
} |
| 266 | 257 |
|
| 267 |
SolrDocumentList docs = rsp.getResults();
|
|
| 258 |
Map<String,List<String>> response = new HashMap<String, List<String>>();
|
|
| 268 | 259 |
|
| 269 |
logger.debug("time: " + rsp.getElapsedTime());
|
|
| 270 |
logger.debug("found: " + docs.getNumFound());
|
|
| 260 |
logger.debug("refine results " + refineSolrResults);
|
|
| 261 |
logger.debug("search results " + searchSolrResults);
|
|
| 271 | 262 |
|
| 272 |
List<String> res = new ArrayList<String>(); |
|
| 263 |
response.put("refine",refineSolrResults);
|
|
| 264 |
response.put("search", searchSolrResults);
|
|
| 273 | 265 |
|
| 274 |
for (int i = 0; i < (to - from) + 1; i++) {
|
|
| 275 |
String result = ((ArrayList<String>) docs.get(i).get("__result")).get(0);
|
|
| 266 |
return response; |
|
| 276 | 267 |
|
| 277 |
res.add(result); |
|
| 278 |
} |
|
| 279 |
|
|
| 280 |
return res; |
|
| 281 |
|
|
| 282 | 268 |
} catch (SolrServerException sse) {
|
| 283 |
logger.error("Fail to get document results.", sse);
|
|
| 269 |
logger.error("Error calling solr", sse);
|
|
| 284 | 270 |
} |
| 285 | 271 |
|
| 286 |
return Collections.EMPTY_LIST;
|
|
| 272 |
return null;
|
|
| 287 | 273 |
} |
| 288 | 274 |
|
| 275 |
|
|
| 289 | 276 |
public static void main(String[] args) throws IOException, CQLParseException, SolrServerException {
|
| 290 | 277 |
CloudSolrServer solrClient = new CloudSolrServer("beta.solr.openaire.eu:9983");
|
| 291 | 278 |
NamedList<String> queryOpts = new NamedList<String>(); |
| ... | ... | |
| 306 | 293 |
queryOpts.add("fq", "popularity");
|
| 307 | 294 |
|
| 308 | 295 |
|
| 296 |
|
|
| 309 | 297 |
// queryOpts.put("fq", new CqlTranslatorImpl().getTranslatedQuery("").asLucene());
|
| 310 | 298 |
// queryOpts.add("facet.field", "contextid");
|
| 311 | 299 |
// queryOpts.add("facet.field", "contextname");
|
Also available in: Unified diff
Added methods for single solr query for search and browse. Size does not trigger extra solr query. Refine uses field queries. Pending APIs for different entities - only the genarla resources api is currently enabled. Pending handling format for both XML and JSON. All these related to #1432 and #2479