Revision 49870
Added by Katerina Iatropoulou almost 7 years ago
| modules/uoa-search/trunk/src/main/java/eu/dnetlib/data/search/solr/SolrResultSet.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.data.search.solr; |
| 2 | 2 |
|
| 3 | 3 |
import com.google.gson.Gson; |
| 4 |
import eu.dnetlib.api.data.SearchServiceException; |
|
| 4 | 5 |
import eu.dnetlib.data.search.transform.Transformer; |
| 5 | 6 |
import eu.dnetlib.data.search.transform.TransformerException; |
| 6 | 7 |
import eu.dnetlib.data.search.utils.solr.SolrResultSetOptionsUtil; |
| 7 | 8 |
import eu.dnetlib.data.search.utils.solr.SolrResultsFormatter; |
| 8 | 9 |
import eu.dnetlib.domain.EPR; |
| 9 |
import eu.dnetlib.functionality.cql.CqlTranslatorImpl; |
|
| 10 | 10 |
import gr.uoa.di.driver.enabling.resultset.ResultSet; |
| 11 | 11 |
import org.apache.commons.lang.StringEscapeUtils; |
| 12 | 12 |
import org.apache.log4j.Logger; |
| ... | ... | |
| 265 | 265 |
} |
| 266 | 266 |
|
| 267 | 267 |
|
| 268 |
public void cursorGet(Transformer transformer, OutputStream os) throws SolrServerException, IOException {
|
|
| 269 |
|
|
| 268 |
public void cursorGet(Transformer transformer, OutputStream os) throws SolrServerException, IOException, SearchServiceException {
|
|
| 270 | 269 |
queryOpts.add("start", "0");
|
| 270 |
queryOpts.add("rows", "0");
|
|
| 271 |
queryOpts.remove("rows");
|
|
| 271 | 272 |
queryOpts.add("rows", "500");
|
| 272 | 273 |
queryOpts.add("fl", "__result");
|
| 273 | 274 |
queryOpts.add("shards.tolerant","true");
|
| ... | ... | |
| 277 | 278 |
String cursorMark = "*"; |
| 278 | 279 |
String nextCursorMark = ""; |
| 279 | 280 |
|
| 281 |
int curs = 0; |
|
| 280 | 282 |
QueryResponse resp = solrClient.query(SolrParams.toSolrParams(queryOpts)); |
| 281 | 283 |
|
| 282 |
int curs = 0; |
|
| 283 | 284 |
while (!cursorMark.equals(nextCursorMark)) {
|
| 284 |
logger.debug("QUERY OPTS: " + queryOpts.get("cursorMark"));
|
|
| 285 | 285 |
resp = solrClient.query(SolrParams.toSolrParams(queryOpts)); |
| 286 |
logger.debug("TOTAL number " + resp.getResults().getNumFound());
|
|
| 287 |
logger.debug(resp.getNextCursorMark()); |
|
| 288 |
|
|
| 289 |
System.out.println("BEGIN");
|
|
| 290 |
System.out.println("cursor " + cursorMark);
|
|
| 291 |
System.out.println("next cursor " + nextCursorMark);
|
|
| 292 |
|
|
| 293 | 286 |
cursorMark = nextCursorMark; |
| 294 | 287 |
nextCursorMark = resp.getNextCursorMark(); |
| 295 | 288 |
|
| ... | ... | |
| 297 | 290 |
if (transformer != null) {
|
| 298 | 291 |
String result = null; |
| 299 | 292 |
try {
|
| 300 |
logger.debug(transformer.getClass()); |
|
| 301 | 293 |
result = transformer.transform(((ArrayList<String>) resp.getResults().get(i).get("__result")).get(0));
|
| 302 |
logger.debug("RESULT " + result);
|
|
| 303 |
} catch (TransformerException e) {
|
|
| 304 |
logger.error("Something went wrong.", e);
|
|
| 294 |
|
|
| 295 |
} catch (TransformerException te) {
|
|
| 296 |
logger.warn("Error transforming " + result, te);
|
|
| 297 |
continue; |
|
| 305 | 298 |
} |
| 306 | 299 |
os.write(result.getBytes()); |
| 307 | 300 |
os.flush(); |
| 308 | 301 |
} |
| 309 | 302 |
} |
| 310 | 303 |
|
| 311 |
|
|
| 312 |
System.out.println("END");
|
|
| 313 |
System.out.println("cursor " + cursorMark);
|
|
| 314 |
System.out.println("next cursor " + nextCursorMark);
|
|
| 315 | 304 |
queryOpts.remove("cursorMark");
|
| 316 | 305 |
queryOpts.add("cursorMark", nextCursorMark);
|
| 317 |
|
|
| 318 |
System.out.println("CURS " + curs);
|
|
| 319 | 306 |
curs ++; |
| 307 |
} |
|
| 320 | 308 |
|
| 321 |
}
|
|
| 309 |
logger.debug("CURS " + curs);
|
|
| 322 | 310 |
} |
| 323 | 311 |
|
| 324 | 312 |
//TODO get rid of this as soon as Joomla portal is out |
| ... | ... | |
| 373 | 361 |
|
| 374 | 362 |
|
| 375 | 363 |
|
| 376 |
public static void main(String[] args) throws IOException, CQLParseException, SolrServerException {
|
|
| 364 |
/* public static void main(String[] args) throws IOException, CQLParseException, SolrServerException {
|
|
| 377 | 365 |
CloudSolrServer solrClient = new CloudSolrServer("beta.solr.openaire.eu:9983");
|
| 378 | 366 |
solrClient.setDefaultCollection("DMF-index-openaire");
|
| 379 | 367 |
|
| ... | ... | |
| 471 | 459 |
|
| 472 | 460 |
System.out.println("max: " + max);
|
| 473 | 461 |
*/ |
| 474 |
} |
|
| 462 |
// }
|
|
| 475 | 463 |
|
| 476 | 464 |
// @Override |
| 477 | 465 |
// public EPR getEpr() {
|
| modules/uoa-search/trunk/src/main/java/eu/dnetlib/data/search/app/SearchServiceImpl.java | ||
|---|---|---|
| 279 | 279 |
EPR epr = null; |
| 280 | 280 |
ResultSet<String> rs = null; |
| 281 | 281 |
|
| 282 |
List<String> browseResults = null; |
|
| 283 |
List<String> searchResults = null; |
|
| 284 |
|
|
| 285 | 282 |
String query = rewrite(text); |
| 286 | 283 |
enhanceFieldQueries(fieldQueries); |
| 287 | 284 |
logger.info("Performing cursor query " + query + "' and fields " + fieldQueries + " and refine " + refinefields);
|
| 285 |
logger.debug("Performing cursor query " + query + "' and fields " + fieldQueries + " and refine " + refinefields);
|
|
| 288 | 286 |
|
| 287 |
|
|
| 289 | 288 |
try {
|
| 290 | 289 |
String eprQuery = createEprQuery(query, refinefields, fieldQueries); |
| 291 | 290 |
epr = index.getBrowsingStatistics(eprQuery, "all", mdFormat, indexLayout); |
| modules/uoa-search/trunk/src/main/java/eu/dnetlib/data/search/utils/vocabulary/IndexVocabularyLoader.java | ||
|---|---|---|
| 34 | 34 |
String xml = null; |
| 35 | 35 |
FormattedSearchResult fsr = null; |
| 36 | 36 |
|
| 37 |
long time = System.currentTimeMillis(); |
|
| 37 |
//long time = System.currentTimeMillis(); |
|
| 38 |
//String stringLocale = locale.getLanguage() + "_" + locale.getCountry(); |
|
| 38 | 39 |
|
| 39 |
//String stringLocale = locale.getLanguage() + "_" + locale.getCountry(); |
|
| 40 | 40 |
try {
|
| 41 | 41 |
fsr = searchServiceLocator.getService().search( |
| 42 | 42 |
((IndexVocabulary)vocabulary).getQuery(), |
| ... | ... | |
| 59 | 59 |
logger.error("Error getting Vocabulary xml", sse);
|
| 60 | 60 |
} |
| 61 | 61 |
|
| 62 |
time = System.currentTimeMillis() - time; |
|
| 62 |
//time = System.currentTimeMillis() - time;
|
|
| 63 | 63 |
//System.out.println("old Vocabulary creation took " + time);
|
| 64 | 64 |
|
| 65 | 65 |
|
| modules/uoa-search/trunk/src/main/java/eu/dnetlib/data/search/web/api/SearchApiService.java | ||
|---|---|---|
| 18 | 18 |
import javax.ws.rs.core.MediaType; |
| 19 | 19 |
import javax.ws.rs.core.Response; |
| 20 | 20 |
import javax.ws.rs.core.StreamingOutput; |
| 21 |
import java.io.*;
|
|
| 22 |
import java.util.HashMap;
|
|
| 21 |
import java.io.IOException;
|
|
| 22 |
import java.io.OutputStream;
|
|
| 23 | 23 |
import java.util.List; |
| 24 | 24 |
import java.util.Locale; |
| 25 |
import java.util.Map; |
|
| 26 |
import java.util.concurrent.TimeUnit; |
|
| 27 | 25 |
|
| 28 | 26 |
/** |
| 29 | 27 |
* Created by kiatrop on 2/6/2016. |
| ... | ... | |
| 575 | 573 |
@QueryParam("fq") final List<String> fieldQueries,
|
| 576 | 574 |
@QueryParam("type") final String type,
|
| 577 | 575 |
@QueryParam("format") final String format,
|
| 578 |
@Context final HttpServletResponse response) throws Exception {
|
|
| 576 |
@Context final HttpServletResponse response) {
|
|
| 579 | 577 |
|
| 580 | 578 |
RequestResponseHandler.Entity entity = extractEntity(type); |
| 581 | 579 |
Transformer transformer = extractTransformer(format, extractEntity(type)); |
| 582 | 580 |
boolean special = isSpecialFormat(format); |
| 583 |
String responseType = extractResponseFormat(format); |
|
| 581 |
String responseType = extractResponseFormat(format);
|
|
| 584 | 582 |
|
| 583 |
try {
|
|
| 584 |
SearchResult sr = searchService.newSearch(query, Locale.getDefault().toString(), fields, fieldQueries,0, 0, format, transformer,null,false); |
|
| 585 |
if (sr.getTotal() > 10000) {
|
|
| 586 |
return Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(APIResponseFormatter. |
|
| 587 |
compose500Message(MediaType.APPLICATION_JSON, "Fail to fetch report.", "You have exceeded the number of allowed returned results")).build(); |
|
| 588 |
} |
|
| 589 |
|
|
| 590 |
} catch (SearchServiceException sse) {
|
|
| 591 |
return Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(APIResponseFormatter. |
|
| 592 |
compose500Message(MediaType.APPLICATION_JSON, "Fail to fetch report.", sse.getMessage())).build(); |
|
| 593 |
} |
|
| 594 |
|
|
| 585 | 595 |
StreamingOutput stream = new StreamingOutput() {
|
| 586 | 596 |
@Override |
| 587 | 597 |
public void write(OutputStream os) throws IOException, WebApplicationException |
| 588 | 598 |
{
|
| 599 |
if (hasTitle(format)) {
|
|
| 600 |
os.write(CSVResponseFormat.appendTitle(entity, special).getBytes()); |
|
| 601 |
} |
|
| 589 | 602 |
try {
|
| 590 |
os.write(CSVResponseFormat.appendTitle(entity, special).getBytes()); |
|
| 591 | 603 |
searchService.cursorSearch(query, fields, fieldQueries, format, transformer, os); |
| 592 | 604 |
|
| 593 | 605 |
} catch (SearchServiceException sse) {
|
| 594 |
os.write(("Fail to return report." + sse.getMessage()).getBytes());
|
|
| 595 |
logger.error("Fail to return report.", sse);
|
|
| 606 |
os.write(("Fail to return report. " + sse.getMessage()).getBytes());
|
|
| 607 |
logger.warn("Fail to return report. ", sse);
|
|
| 596 | 608 |
} |
| 609 |
|
|
| 597 | 610 |
os.close(); |
| 598 | 611 |
} |
| 599 | 612 |
|
| 600 | 613 |
}; |
| 601 | 614 |
|
| 602 |
return Response.ok().entity(stream).type( MediaType.TEXT_PLAIN).build(); |
|
| 615 |
return Response.ok().entity(stream).type(responseType).build(); |
|
| 616 |
} |
|
| 603 | 617 |
|
| 604 |
|
|
| 605 |
//q=*:*&start=0&rows=10&cursorMark=*&sort=dateofcollection asc |
|
| 606 |
/* try {
|
|
| 607 |
queryOpts.add("q", new CqlTranslatorImpl().getTranslatedQuery("(oaftype exact result) and (resulttypeid exact publication)").asLucene());
|
|
| 608 |
|
|
| 609 |
} catch (CQLParseException e) {
|
|
| 610 |
logger.error(e); |
|
| 611 |
} |
|
| 612 |
queryOpts.add("start", "0");
|
|
| 613 |
queryOpts.add("rows", "500");
|
|
| 614 |
queryOpts.add("fl", "__result");
|
|
| 615 |
queryOpts.add("shards.tolerant","true");
|
|
| 616 |
queryOpts.add("cursorMark", "*");
|
|
| 617 |
queryOpts.add("sort", "__indexrecordidentifier asc");
|
|
| 618 |
|
|
| 619 |
StreamingOutput stream = new StreamingOutput() {
|
|
| 620 |
@Override |
|
| 621 |
public void write(OutputStream os) throws IOException, WebApplicationException |
|
| 622 |
{
|
|
| 623 |
int curs = 0; |
|
| 624 |
QueryResponse resp = null; |
|
| 625 |
|
|
| 626 |
String cursorMark = "*"; |
|
| 627 |
String nextCursorMark = ""; |
|
| 628 |
|
|
| 629 |
while (!cursorMark.equals(nextCursorMark)) {
|
|
| 630 |
logger.debug("QUERY OPTS: " + queryOpts.get("cursorMark"));
|
|
| 631 |
try {
|
|
| 632 |
resp = solrClient.query(SolrParams.toSolrParams(queryOpts)); |
|
| 633 |
} catch (SolrServerException e) {
|
|
| 634 |
logger.error("ERROROROROROROR!! ", e);
|
|
| 635 |
e.printStackTrace(); |
|
| 636 |
} |
|
| 637 |
logger.debug("TOTAL number " + resp.getResults().getNumFound());
|
|
| 638 |
logger.debug(resp.getNextCursorMark()); |
|
| 639 |
|
|
| 640 |
System.out.println("BEGIN");
|
|
| 641 |
System.out.println("cursor " + cursorMark);
|
|
| 642 |
System.out.println("next cursor " + nextCursorMark);
|
|
| 643 |
|
|
| 644 |
cursorMark = nextCursorMark; |
|
| 645 |
nextCursorMark = resp.getNextCursorMark(); |
|
| 646 |
|
|
| 647 |
for (int i = 0; i < resp.getResults().size(); i++) {
|
|
| 648 |
String result = ((ArrayList<String>) resp.getResults().get(i).get("__result")).get(0);
|
|
| 649 |
os.write(result.getBytes()); |
|
| 650 |
os.flush(); |
|
| 651 |
} |
|
| 652 |
|
|
| 653 |
System.out.println("END");
|
|
| 654 |
System.out.println("cursor " + cursorMark);
|
|
| 655 |
System.out.println("next cursor " + nextCursorMark);
|
|
| 656 |
queryOpts.remove("cursorMark");
|
|
| 657 |
queryOpts.add("cursorMark", nextCursorMark);
|
|
| 658 |
|
|
| 659 |
System.out.println("CURS " + curs);
|
|
| 660 |
curs ++; |
|
| 661 |
|
|
| 662 |
} |
|
| 663 |
} |
|
| 664 |
}; |
|
| 665 |
|
|
| 666 |
//out.close(); |
|
| 667 |
|
|
| 668 |
time = System.currentTimeMillis() - time; |
|
| 669 |
System.out.println("Answer time " + time);
|
|
| 670 |
//StreamingOutput output = response.getWriter(); |
|
| 671 |
return Response.ok().entity( stream ).type( MediaType.TEXT_PLAIN).build();*/ |
|
| 672 |
//return Response.status(Response.Status.OK).build(); |
|
| 673 |
} |
|
| 674 |
|
|
| 675 |
|
|
| 676 |
@GET |
|
| 677 |
@Path("/testreports")
|
|
| 678 |
@Produces(MediaType.TEXT_PLAIN) |
|
| 679 |
public Response loadHierarchy(@PathParam( "pkPerson" ) String pkPerson) {
|
|
| 680 |
final Map<Integer, String> people = new HashMap<>(); |
|
| 681 |
people.put(1, "Michael\n"); |
|
| 682 |
people.put(2, "Mark\n"); |
|
| 683 |
|
|
| 684 |
StreamingOutput stream = new StreamingOutput() {
|
|
| 685 |
@Override |
|
| 686 |
public void write(OutputStream os) throws IOException, WebApplicationException |
|
| 687 |
{
|
|
| 688 |
|
|
| 689 |
while (true) {
|
|
| 690 |
Writer writer = new BufferedWriter(new OutputStreamWriter(os)); |
|
| 691 |
writer.write(people.get(1)); |
|
| 692 |
writer.flush(); |
|
| 693 |
try {
|
|
| 694 |
TimeUnit.SECONDS.sleep(1); |
|
| 695 |
} catch (InterruptedException e) {
|
|
| 696 |
e.printStackTrace(); |
|
| 697 |
logger.error("ERROR.", e);
|
|
| 698 |
} |
|
| 699 |
} |
|
| 700 |
|
|
| 701 |
/*while(true) {
|
|
| 702 |
os.write(people.get(1).getBytes()); |
|
| 703 |
os.flush(); |
|
| 704 |
}*/ |
|
| 705 |
} |
|
| 706 |
}; |
|
| 707 |
|
|
| 708 |
return Response.ok().entity( stream ).type( MediaType.TEXT_PLAIN).build() ; |
|
| 709 |
} |
|
| 710 |
|
|
| 711 | 618 |
private String extractResponseFormat(String format) {
|
| 712 | 619 |
if (format != null && !format.isEmpty()) {
|
| 713 | 620 |
if (format.equalsIgnoreCase("json")) {
|
| modules/uoa-search/trunk/src/main/java/eu/dnetlib/data/search/web/api/SearchRequestController.java | ||
|---|---|---|
| 238 | 238 |
} |
| 239 | 239 |
|
| 240 | 240 |
time = System.currentTimeMillis() - time; |
| 241 |
System.out.println("Answer old time " + time);
|
|
| 241 |
logger.debug("Answer old time " + time);
|
|
| 242 | 242 |
} |
| 243 | 243 |
|
| 244 | 244 |
@RequestMapping(value = "/api/datasets", method = RequestMethod.GET) |
Also available in: Unified diff
Added check for allowed number of returned results in cursor search