«
Previous
|
Next
»
Revision 63331
Added by Katerina Iatropoulou 5 days ago
| modules/uoa-search/branches/uoa-search-json/pom.xml | ||
|---|---|---|
| 22 | 22 |
</snapshots> |
| 23 | 23 |
<id>cloudera-central</id> |
| 24 | 24 |
<name>cloundera-libs-release</name> |
| 25 |
<url>http://maven.research-infrastructures.eu/nexus/content/repositories/cloudera-central</url> |
|
| 25 |
<url>https://maven.research-infrastructures.eu/nexus/content/repositories/cloudera-central</url>
|
|
| 26 | 26 |
</repository> |
| 27 | 27 |
<repository> |
| 28 | 28 |
<id>cloudera-snapshots</id> |
| 29 | 29 |
<name>cloudera-libs-snapshot</name> |
| 30 |
<url>http://maven.research-infrastructures.eu/nexus/content/repositories/cloudera-snapshots</url> |
|
| 30 |
<url>https://maven.research-infrastructures.eu/nexus/content/repositories/cloudera-snapshots</url>
|
|
| 31 | 31 |
</repository> |
| 32 | 32 |
<repository> |
| 33 | 33 |
<id>typesafe</id> |
| 34 | 34 |
<name>typesafe-releases</name> |
| 35 |
<url>http://maven.research-infrastructures.eu/nexus/content/repositories/typesafe</url> |
|
| 35 |
<url>https://maven.research-infrastructures.eu/nexus/content/repositories/typesafe</url>
|
|
| 36 | 36 |
</repository> |
| 37 |
<repository> |
|
| 38 |
<id>dnet45-releases</id> |
|
| 39 |
<name>D-Net 45 releases</name> |
|
| 40 |
<url>https://maven.d4science.org/nexus/content/repositories/dnet45-releases</url> |
|
| 41 |
<layout>default</layout> |
|
| 42 |
<snapshots> |
|
| 43 |
<enabled>false</enabled> |
|
| 44 |
</snapshots> |
|
| 45 |
<releases> |
|
| 46 |
<enabled>true</enabled> |
|
| 47 |
</releases> |
|
| 48 |
</repository> |
|
| 49 |
<repository> |
|
| 50 |
<id>dnet45-snapshots</id> |
|
| 51 |
<name>D-Net 45 snapshots</name> |
|
| 52 |
<url>https://maven.d4science.org/nexus/content/repositories/dnet45-snapshots</url> |
|
| 53 |
<layout>default</layout> |
|
| 54 |
<snapshots> |
|
| 55 |
<enabled>true</enabled> |
|
| 56 |
</snapshots> |
|
| 57 |
<releases> |
|
| 58 |
<enabled>false</enabled> |
|
| 59 |
</releases> |
|
| 60 |
</repository> |
|
| 61 |
<repository> |
|
| 62 |
<id>dnet45-bootstrap-snapshot</id> |
|
| 63 |
<name>D-Net 45 Bootstrap Snapshot</name> |
|
| 64 |
<url>https://maven.d4science.org/nexus/content/repositories/dnet45-bootstrap-snapshot/</url> |
|
| 65 |
<releases> |
|
| 66 |
<enabled>false</enabled> |
|
| 67 |
</releases> |
|
| 68 |
<snapshots> |
|
| 69 |
<enabled>true</enabled> |
|
| 70 |
</snapshots> |
|
| 71 |
<layout>default</layout> |
|
| 72 |
</repository> |
|
| 73 |
<repository> |
|
| 74 |
<id>dnet45-bootstrap-release</id> |
|
| 75 |
<name>D-Net 45 Bootstrap Release</name> |
|
| 76 |
<url>https://maven.d4science.org/nexus/content/repositories/dnet45-bootstrap-release/</url> |
|
| 77 |
<releases> |
|
| 78 |
<enabled>true</enabled> |
|
| 79 |
</releases> |
|
| 80 |
<snapshots> |
|
| 81 |
<enabled>false</enabled> |
|
| 82 |
</snapshots> |
|
| 83 |
<layout>default</layout> |
|
| 84 |
</repository> |
|
| 37 | 85 |
</repositories> |
| 38 | 86 |
|
| 39 | 87 |
<dependencies> |
| ... | ... | |
| 230 | 278 |
<version>3.3.9</version> |
| 231 | 279 |
</dependency> |
| 232 | 280 |
|
| 281 |
<!-- CNR for entities --> |
|
| 282 |
<dependency> |
|
| 283 |
<groupId>eu.dnetlib.dhp</groupId> |
|
| 284 |
<artifactId>dhp-schemas</artifactId> |
|
| 285 |
<version>[6.1.1]</version> |
|
| 286 |
</dependency> |
|
| 233 | 287 |
|
| 288 |
<dependency> |
|
| 289 |
<groupId>eu.dnetlib.dhp</groupId> |
|
| 290 |
<artifactId>dhp-dump-schema</artifactId> |
|
| 291 |
<version>[8.0.0-SNAPSHOT]</version> |
|
| 292 |
</dependency> |
|
| 234 | 293 |
|
| 294 |
<dependency> |
|
| 295 |
<groupId>com.fasterxml.jackson.core</groupId> |
|
| 296 |
<artifactId>jackson-core</artifactId> |
|
| 297 |
<version>2.15.2</version> |
|
| 298 |
</dependency> |
|
| 299 |
<dependency> |
|
| 300 |
<groupId>com.fasterxml.jackson.core</groupId> |
|
| 301 |
<artifactId>jackson-databind</artifactId> |
|
| 302 |
<version>2.15.2</version> |
|
| 303 |
</dependency> |
|
| 304 |
<dependency> |
|
| 305 |
<groupId>com.fasterxml.jackson.core</groupId> |
|
| 306 |
<artifactId>jackson-annotations</artifactId> |
|
| 307 |
<version>2.15.2</version> |
|
| 308 |
</dependency> |
|
| 309 |
<dependency> |
|
| 310 |
<groupId>com.fasterxml.jackson.dataformat</groupId> |
|
| 311 |
<artifactId>jackson-dataformat-csv</artifactId> |
|
| 312 |
<version>2.15.2</version> |
|
| 313 |
</dependency> |
|
| 314 |
<dependency> |
|
| 315 |
<groupId>org.mapstruct</groupId> |
|
| 316 |
<artifactId>mapstruct</artifactId> |
|
| 317 |
<version>1.5.5.Final</version> |
|
| 318 |
</dependency> |
|
| 319 |
|
|
| 320 |
<dependency> |
|
| 321 |
<groupId>org.mapstruct</groupId> |
|
| 322 |
<artifactId>mapstruct</artifactId> |
|
| 323 |
<version>1.5.5.Final</version> |
|
| 324 |
</dependency> |
|
| 325 |
<dependency> |
|
| 326 |
<groupId>org.mapstruct</groupId> |
|
| 327 |
<artifactId>mapstruct-processor</artifactId> |
|
| 328 |
<version>1.5.5.Final</version> |
|
| 329 |
<scope>provided</scope> |
|
| 330 |
</dependency> |
|
| 331 |
|
|
| 332 |
<dependency> |
|
| 333 |
<groupId>com.fasterxml.jackson.core</groupId> |
|
| 334 |
<artifactId>jackson-core</artifactId> |
|
| 335 |
<version>2.15.2</version> |
|
| 336 |
</dependency> |
|
| 337 |
<dependency> |
|
| 338 |
<groupId>com.fasterxml.jackson.core</groupId> |
|
| 339 |
<artifactId>jackson-databind</artifactId> |
|
| 340 |
<version>2.15.2</version> |
|
| 341 |
</dependency> |
|
| 342 |
<dependency> |
|
| 343 |
<groupId>com.fasterxml.jackson.core</groupId> |
|
| 344 |
<artifactId>jackson-annotations</artifactId> |
|
| 345 |
<version>2.15.2</version> |
|
| 346 |
</dependency> |
|
| 347 |
<dependency> |
|
| 348 |
<groupId>com.fasterxml.jackson.dataformat</groupId> |
|
| 349 |
<artifactId>jackson-dataformat-csv</artifactId> |
|
| 350 |
<version>2.15.2</version> |
|
| 351 |
</dependency> |
|
| 352 |
|
|
| 235 | 353 |
</dependencies> |
| 236 | 354 |
|
| 237 | 355 |
<build> |
| modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/app/SearchServiceImpl.java | ||
|---|---|---|
| 343 | 343 |
return new SearchResult(query, Locale.getDefault().toString(), rs.size(), from, to, searchResults, browseResults, refinefields); |
| 344 | 344 |
} |
| 345 | 345 |
|
| 346 |
public SearchResult newJsonSearch (String text, String locale, List<String> refinefields, List<String> specialFacets, List<String> fieldQueries, |
|
| 347 |
int from, int to, String format, Transformer transformer, Transformer oldRefineTransformer, |
|
| 348 |
boolean oldPaging, boolean minRef, boolean timeout) throws SearchServiceException {
|
|
| 349 |
long startTime = System.currentTimeMillis(); |
|
| 350 |
Timer.Sample timer = Timer.start(registry); |
|
| 351 |
|
|
| 352 |
IndexService index = getIndexLocator().getService(); |
|
| 353 |
|
|
| 354 |
EPR epr = null; |
|
| 355 |
ResultSet<String> rs = null; |
|
| 356 |
|
|
| 357 |
List<String> browseResults = null; |
|
| 358 |
List<String> searchResults = null; |
|
| 359 |
|
|
| 360 |
String query = rewrite(text); |
|
| 361 |
enhanceFieldQueries(fieldQueries); |
|
| 362 |
logger.info("Performing query '" + query + "' and fields " + fieldQueries + " and refine " + refinefields);
|
|
| 363 |
|
|
| 364 |
try {
|
|
| 365 |
//TODO see parser and maybe delete! |
|
| 366 |
//query = new CQLParser().parse(query).toCQL(); |
|
| 367 |
String eprQuery = createEprQuery(query, refinefields, specialFacets, fieldQueries, minRef); |
|
| 368 |
//System.out.println("The eprQuery is now " + eprQuery);
|
|
| 369 |
|
|
| 370 |
epr = index.getBrowsingStatistics(eprQuery, "all", mdFormat, indexLayout); |
|
| 371 |
|
|
| 372 |
if (epr == null){
|
|
| 373 |
throw new SearchServiceException("Something really strange happened there! Index returned null result set id.");
|
|
| 374 |
} |
|
| 375 |
|
|
| 376 |
//get the locale TODO do we need this? |
|
| 377 |
//String correctLocale = getCorrectLocale(locale); |
|
| 378 |
//StringTokenizer tokenizer = new StringTokenizer(correctLocale, "_"); |
|
| 379 |
//Locale requestLocale = new Locale(tokenizer.nextToken(), tokenizer.nextToken()); |
|
| 380 |
|
|
| 381 |
|
|
| 382 |
|
|
| 383 |
if (oldPaging) {
|
|
| 384 |
rs = ((SolrResultSetFactory) rsFactory).createResultSetWithoutSocketTimeout(epr); |
|
| 385 |
} else {
|
|
| 386 |
rs = ((SolrResultSetFactory)rsFactory).createResultSetWithSocketTimeout(epr); |
|
| 387 |
} |
|
| 388 |
|
|
| 389 |
if (rs == null ) {
|
|
| 390 |
logger.info("Rs is null. Search will return an error.");
|
|
| 391 |
logger.error("Received null result set.");
|
|
| 392 |
throw new SearchServiceException("Error getting search results from index.");
|
|
| 393 |
} |
|
| 394 |
|
|
| 395 |
Map<String, List<String>> list = null; |
|
| 396 |
if (oldPaging) { // for the public API calls
|
|
| 397 |
list = ((SolrResultSet)rs).newJsonGet(from-1, to, format, transformer, oldRefineTransformer, false, timeoutValue); |
|
| 398 |
|
|
| 399 |
} else {
|
|
| 400 |
list = ((SolrResultSet)rs).newJsonGet(from, to, format, transformer, oldRefineTransformer, true, timeoutValue); |
|
| 401 |
} |
|
| 402 |
|
|
| 403 |
searchResults = list.get("search");
|
|
| 404 |
browseResults = list.get("refine");
|
|
| 405 |
|
|
| 406 |
} catch (IndexServiceException ise) {
|
|
| 407 |
logger.error("Error getting refine results.", ise);
|
|
| 408 |
throw new SearchServiceException("Error getting refine results.", ise);
|
|
| 409 |
|
|
| 410 |
} catch (CQLParseException e) {
|
|
| 411 |
logger.error("CQL Parse Exception.");
|
|
| 412 |
throw new SearchServiceException("Syntax errors. " + e.getMessage());
|
|
| 413 |
|
|
| 414 |
} catch (IOException e) {
|
|
| 415 |
throw new SearchServiceException("Error getting results. ", e);
|
|
| 416 |
|
|
| 417 |
} finally {
|
|
| 418 |
timer.stop(registry.timer("search.server.response.duration"));
|
|
| 419 |
} |
|
| 420 |
|
|
| 421 |
long estimatedTime = System.currentTimeMillis() - startTime; |
|
| 422 |
|
|
| 423 |
logger.debug("Inner search time " + estimatedTime + " milliseconds for query " + query +
|
|
| 424 |
" and fields " + fieldQueries + " and refine " + refinefields + " from: "+ from + " and size " + to); |
|
| 425 |
|
|
| 426 |
rs.close(); |
|
| 427 |
|
|
| 428 |
return new SearchResult(query, Locale.getDefault().toString(), rs.size(), from, to, searchResults, browseResults, refinefields); |
|
| 429 |
} |
|
| 430 |
|
|
| 346 | 431 |
public void cursorSearch(String text, List<String> refinefields, List<String> specialFacets, List<String> fieldQueries, |
| 347 |
String format, Transformer transformer, OutputStream os, int limit) throws SearchServiceException {
|
|
| 432 |
String format, Transformer transformer, OutputStream os, boolean special, boolean hasTitle, boolean isHtml, int limit) throws SearchServiceException {
|
|
| 348 | 433 |
|
| 349 | 434 |
long startTime = System.nanoTime(); |
| 350 | 435 |
|
| ... | ... | |
| 375 | 460 |
throw new SearchServiceException("Error getting search results from index.");
|
| 376 | 461 |
} |
| 377 | 462 |
|
| 378 |
((SolrResultSet)rs).cursorGet(transformer, limit, os); |
|
| 463 |
((SolrResultSet)rs).cursorGet(transformer, special, hasTitle, isHtml, limit, os);
|
|
| 379 | 464 |
|
| 380 | 465 |
} catch (IndexServiceException ise) {
|
| 381 | 466 |
logger.error("Error getting cursor results.", ise);
|
| ... | ... | |
| 546 | 631 |
Map<String, List<String>> list = null; |
| 547 | 632 |
|
| 548 | 633 |
|
| 634 |
//TODO check this for Json |
|
| 549 | 635 |
if (oldPaging) {
|
| 550 |
list = ((SolrResultSet)rs).newGet(from-1, to, format, transformer, oldRefineTransformer, timeout, timeoutValue); |
|
| 636 |
list = ((SolrResultSet)rs).newJsonGet(from-1, to, format, transformer, oldRefineTransformer, timeout, timeoutValue);
|
|
| 551 | 637 |
|
| 552 | 638 |
} else {
|
| 553 |
list = ((SolrResultSet)rs).newGet(from, to, format, transformer, oldRefineTransformer, timeout, timeoutValue); |
|
| 639 |
list = ((SolrResultSet)rs).newJsonGet(from, to, format, transformer, oldRefineTransformer, timeout, timeoutValue);
|
|
| 554 | 640 |
} |
| 555 | 641 |
|
| 556 | 642 |
|
| modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/helpers/csv/CsvEntity.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.search.helpers.csv; |
|
| 2 |
|
|
| 3 |
public interface CsvEntity {
|
|
| 4 |
|
|
| 5 |
} |
|
| modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/helpers/csv/DatasourceCsv.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.search.helpers.csv; |
|
| 2 |
|
|
| 3 |
import com.fasterxml.jackson.annotation.JsonProperty; |
|
| 4 |
import com.fasterxml.jackson.annotation.JsonPropertyOrder; |
|
| 5 |
|
|
| 6 |
import java.util.List; |
|
| 7 |
|
|
| 8 |
@JsonPropertyOrder({ "name", "type", "compatibility", "oaipmh", "websiteUrl" })
|
|
| 9 |
public class DatasourceCsv implements CsvEntity {
|
|
| 10 |
|
|
| 11 |
@JsonProperty("Νame")
|
|
| 12 |
String name; |
|
| 13 |
@JsonProperty("Type")
|
|
| 14 |
String type; |
|
| 15 |
@JsonProperty("Compatibility")
|
|
| 16 |
String compatibility; |
|
| 17 |
@JsonProperty("OAI-PMH")
|
|
| 18 |
List<String> oaipmh; |
|
| 19 |
@JsonProperty("Website")
|
|
| 20 |
String websiteUrl; |
|
| 21 |
|
|
| 22 |
public String getName() {
|
|
| 23 |
return name; |
|
| 24 |
} |
|
| 25 |
|
|
| 26 |
public void setName(String name) {
|
|
| 27 |
this.name = name; |
|
| 28 |
} |
|
| 29 |
|
|
| 30 |
public String getType() {
|
|
| 31 |
return type; |
|
| 32 |
} |
|
| 33 |
|
|
| 34 |
public void setType(String type) {
|
|
| 35 |
this.type = type; |
|
| 36 |
} |
|
| 37 |
|
|
| 38 |
public String getCompatibility() {
|
|
| 39 |
return compatibility; |
|
| 40 |
} |
|
| 41 |
|
|
| 42 |
public void setCompatibility(String compatibility) {
|
|
| 43 |
this.compatibility = compatibility; |
|
| 44 |
} |
|
| 45 |
|
|
| 46 |
public List<String> getOaipmh() {
|
|
| 47 |
return oaipmh; |
|
| 48 |
} |
|
| 49 |
|
|
| 50 |
public void setOaipmh(List<String> oaipmh) {
|
|
| 51 |
this.oaipmh = oaipmh; |
|
| 52 |
} |
|
| 53 |
|
|
| 54 |
public String getWebsiteUrl() {
|
|
| 55 |
return websiteUrl; |
|
| 56 |
} |
|
| 57 |
|
|
| 58 |
public void setWebsiteUrl(String websiteUrl) {
|
|
| 59 |
this.websiteUrl = websiteUrl; |
|
| 60 |
} |
|
| 61 |
} |
|
| modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/helpers/csv/OrganizationCsv.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.search.helpers.csv; |
|
| 2 |
|
|
| 3 |
import com.fasterxml.jackson.annotation.JsonProperty; |
|
| 4 |
import com.fasterxml.jackson.annotation.JsonPropertyOrder; |
|
| 5 |
|
|
| 6 |
@JsonPropertyOrder({ "name", "country"})
|
|
| 7 |
public class OrganizationCsv implements CsvEntity {
|
|
| 8 |
|
|
| 9 |
@JsonProperty("Name")
|
|
| 10 |
String name; |
|
| 11 |
@JsonProperty("Country")
|
|
| 12 |
String country; |
|
| 13 |
|
|
| 14 |
@Override |
|
| 15 |
public String toString() {
|
|
| 16 |
return "OrganizationCsv{" +
|
|
| 17 |
"name='" + name + '\'' + |
|
| 18 |
", country='" + country + '\'' + |
|
| 19 |
'}'; |
|
| 20 |
} |
|
| 21 |
|
|
| 22 |
public String getName() {
|
|
| 23 |
return name; |
|
| 24 |
} |
|
| 25 |
|
|
| 26 |
public void setName(String name) {
|
|
| 27 |
this.name = name; |
|
| 28 |
} |
|
| 29 |
|
|
| 30 |
public String getCountry() {
|
|
| 31 |
return country; |
|
| 32 |
} |
|
| 33 |
|
|
| 34 |
public void setCountry(String country) {
|
|
| 35 |
this.country = country; |
|
| 36 |
} |
|
| 37 |
} |
|
| modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/helpers/csv/ProjectCsv.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.search.helpers.csv; |
|
| 2 |
|
|
| 3 |
import com.fasterxml.jackson.annotation.JsonProperty; |
|
| 4 |
import com.fasterxml.jackson.annotation.JsonPropertyOrder; |
|
| 5 |
|
|
| 6 |
import java.util.List; |
|
| 7 |
|
|
| 8 |
@JsonPropertyOrder({"title", "funder", "startDate", "endDate", "participants"})
|
|
| 9 |
public class ProjectCsv implements CsvEntity {
|
|
| 10 |
//Title Funder Start Date End Date Participants |
|
| 11 |
@JsonProperty("Tilte")
|
|
| 12 |
String title; |
|
| 13 |
@JsonProperty("Funder")
|
|
| 14 |
String funder; |
|
| 15 |
@JsonProperty("Start date")
|
|
| 16 |
String startDate; |
|
| 17 |
@JsonProperty("End date")
|
|
| 18 |
String endDate; |
|
| 19 |
@JsonProperty("Participants")
|
|
| 20 |
List<String> participants; |
|
| 21 |
|
|
| 22 |
public String getTitle() {
|
|
| 23 |
return title; |
|
| 24 |
} |
|
| 25 |
|
|
| 26 |
public void setTitle(String title) {
|
|
| 27 |
this.title = title; |
|
| 28 |
} |
|
| 29 |
|
|
| 30 |
public String getFunder() {
|
|
| 31 |
return funder; |
|
| 32 |
} |
|
| 33 |
|
|
| 34 |
public void setFunder(String funder) {
|
|
| 35 |
this.funder = funder; |
|
| 36 |
} |
|
| 37 |
|
|
| 38 |
public String getStartDate() {
|
|
| 39 |
return startDate; |
|
| 40 |
} |
|
| 41 |
|
|
| 42 |
public void setStartDate(String startDate) {
|
|
| 43 |
this.startDate = startDate; |
|
| 44 |
} |
|
| 45 |
|
|
| 46 |
public String getEndDate() {
|
|
| 47 |
return endDate; |
|
| 48 |
} |
|
| 49 |
|
|
| 50 |
public void setEndDate(String endDate) {
|
|
| 51 |
this.endDate = endDate; |
|
| 52 |
} |
|
| 53 |
|
|
| 54 |
public List<String> getParticipants() {
|
|
| 55 |
return participants; |
|
| 56 |
} |
|
| 57 |
|
|
| 58 |
public void setParticipants(List<String> participants) {
|
|
| 59 |
this.participants = participants; |
|
| 60 |
} |
|
| 61 |
|
|
| 62 |
} |
|
| modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/helpers/csv/ResearchProductCsv.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.search.helpers.csv; |
|
| 2 |
|
|
| 3 |
import com.fasterxml.jackson.annotation.JsonProperty; |
|
| 4 |
import com.fasterxml.jackson.annotation.JsonPropertyOrder; |
|
| 5 |
import lombok.Data; |
|
| 6 |
|
|
| 7 |
import java.util.List; |
|
| 8 |
|
|
| 9 |
//Type Title Authors Publication Year DOI Type Peer Reviewed Diamond Journal Journal Download from Publicly Funded Funder|Project Name (GA Number) Access Green OA colour |
|
| 10 |
|
|
| 11 |
@Data |
|
| 12 |
@JsonPropertyOrder({"researchProduct", "title", "authors", "publicationYear", "doi", "type", "peerReviewed",
|
|
| 13 |
"diamondJournal", "downloadFrom", "publiclyFunded", "funder_Project", "access", "green", "oaColour"}) |
|
| 14 |
public class ResearchProductCsv implements CsvEntity {
|
|
| 15 |
|
|
| 16 |
@JsonProperty("Research Product Type")
|
|
| 17 |
String researchProduct; |
|
| 18 |
@JsonProperty("Title")
|
|
| 19 |
String title; |
|
| 20 |
@JsonProperty("Authors")
|
|
| 21 |
List<String> authors; |
|
| 22 |
@JsonProperty("Publication Year")
|
|
| 23 |
String publicationYear; |
|
| 24 |
@JsonProperty("DOI")
|
|
| 25 |
List<String> doi; |
|
| 26 |
@JsonProperty("Type")
|
|
| 27 |
String type; |
|
| 28 |
@JsonProperty("Peer Reviewed")
|
|
| 29 |
String peerReviewed; |
|
| 30 |
@JsonProperty("Diamond Journal")
|
|
| 31 |
boolean diamondJournal; |
|
| 32 |
@JsonProperty("Journal")
|
|
| 33 |
String journal; |
|
| 34 |
@JsonProperty("Download From")
|
|
| 35 |
String downloadFrom; |
|
| 36 |
@JsonProperty("Publicly Funded")
|
|
| 37 |
boolean publiclyFunded; |
|
| 38 |
//TODO fix this |
|
| 39 |
@JsonProperty("Funder|Project Name (GA Number)")
|
|
| 40 |
String funder_Project; |
|
| 41 |
@JsonProperty("Access")
|
|
| 42 |
String access; |
|
| 43 |
@JsonProperty("Green")
|
|
| 44 |
boolean green; |
|
| 45 |
@JsonProperty("OA colour")
|
|
| 46 |
String oaColour; |
|
| 47 |
|
|
| 48 |
public String getResearchProduct() {
|
|
| 49 |
return researchProduct; |
|
| 50 |
} |
|
| 51 |
|
|
| 52 |
public void setResearchProduct(String researchProduct) {
|
|
| 53 |
this.researchProduct = researchProduct; |
|
| 54 |
} |
|
| 55 |
|
|
| 56 |
public String getTitle() {
|
|
| 57 |
return title; |
|
| 58 |
} |
|
| 59 |
|
|
| 60 |
public void setTitle(String title) {
|
|
| 61 |
this.title = title; |
|
| 62 |
} |
|
| 63 |
|
|
| 64 |
public List<String> getAuthors() {
|
|
| 65 |
return authors; |
|
| 66 |
} |
|
| 67 |
|
|
| 68 |
public void setAuthors(List<String> authors) {
|
|
| 69 |
this.authors = authors; |
|
| 70 |
} |
|
| 71 |
|
|
| 72 |
public String getPublicationYear() {
|
|
| 73 |
return publicationYear; |
|
| 74 |
} |
|
| 75 |
|
|
| 76 |
public void setPublicationYear(String publicationYear) {
|
|
| 77 |
this.publicationYear = publicationYear; |
|
| 78 |
} |
|
| 79 |
|
|
| 80 |
public List<String> getDoi() {
|
|
| 81 |
return doi; |
|
| 82 |
} |
|
| 83 |
|
|
| 84 |
public void setDoi(List<String> doi) {
|
|
| 85 |
this.doi = doi; |
|
| 86 |
} |
|
| 87 |
|
|
| 88 |
public String getType() {
|
|
| 89 |
return type; |
|
| 90 |
} |
|
| 91 |
|
|
| 92 |
public void setType(String type) {
|
|
| 93 |
this.type = type; |
|
| 94 |
} |
|
| 95 |
|
|
| 96 |
public String getPeerReviewed() {
|
|
| 97 |
return peerReviewed; |
|
| 98 |
} |
|
| 99 |
|
|
| 100 |
public void setPeerReviewed(String peerReviewed) {
|
|
| 101 |
this.peerReviewed = peerReviewed; |
|
| 102 |
} |
|
| 103 |
|
|
| 104 |
public boolean isDiamondJournal() {
|
|
| 105 |
return diamondJournal; |
|
| 106 |
} |
|
| 107 |
|
|
| 108 |
public void setDiamondJournal(boolean diamondJournal) {
|
|
| 109 |
this.diamondJournal = diamondJournal; |
|
| 110 |
} |
|
| 111 |
|
|
| 112 |
public String getJournal() {
|
|
| 113 |
return journal; |
|
| 114 |
} |
|
| 115 |
|
|
| 116 |
public void setJournal(String journal) {
|
|
| 117 |
this.journal = journal; |
|
| 118 |
} |
|
| 119 |
|
|
| 120 |
public String getDownloadFrom() {
|
|
| 121 |
return downloadFrom; |
|
| 122 |
} |
|
| 123 |
|
|
| 124 |
public void setDownloadFrom(String downloadFrom) {
|
|
| 125 |
this.downloadFrom = downloadFrom; |
|
| 126 |
} |
|
| 127 |
|
|
| 128 |
public boolean isPubliclyFunded() {
|
|
| 129 |
return publiclyFunded; |
|
| 130 |
} |
|
| 131 |
|
|
| 132 |
public void setPubliclyFunded(boolean publiclyFunded) {
|
|
| 133 |
this.publiclyFunded = publiclyFunded; |
|
| 134 |
} |
|
| 135 |
|
|
| 136 |
public String getFunder_Project() {
|
|
| 137 |
return funder_Project; |
|
| 138 |
} |
|
| 139 |
|
|
| 140 |
public void setFunder_Project(String funder_Project) {
|
|
| 141 |
this.funder_Project = funder_Project; |
|
| 142 |
} |
|
| 143 |
|
|
| 144 |
public String getAccess() {
|
|
| 145 |
return access; |
|
| 146 |
} |
|
| 147 |
|
|
| 148 |
public void setAccess(String access) {
|
|
| 149 |
this.access = access; |
|
| 150 |
} |
|
| 151 |
|
|
| 152 |
public boolean isGreen() {
|
|
| 153 |
return green; |
|
| 154 |
} |
|
| 155 |
|
|
| 156 |
public void setGreen(boolean green) {
|
|
| 157 |
this.green = green; |
|
| 158 |
} |
|
| 159 |
|
|
| 160 |
public String getOaColour() {
|
|
| 161 |
return oaColour; |
|
| 162 |
} |
|
| 163 |
|
|
| 164 |
public void setOaColour(String oaColour) {
|
|
| 165 |
this.oaColour = oaColour; |
|
| 166 |
} |
|
| 167 |
} |
|
| modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/helpers/csv/ResearchProductCsvConverter.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.search.helpers.csv; |
|
| 2 |
|
|
| 3 |
import eu.dnetlib.data.search.mappers.csv.ResearchProductCsvMapper; |
|
| 4 |
import eu.dnetlib.dhp.schema.solr.RelatedRecord; |
|
| 5 |
import eu.dnetlib.dhp.schema.solr.SolrRecord; |
|
| 6 |
import org.springframework.beans.factory.annotation.Autowired; |
|
| 7 |
import org.springframework.stereotype.Component; |
|
| 8 |
|
|
| 9 |
import java.util.ArrayList; |
|
| 10 |
import java.util.List; |
|
| 11 |
import java.util.Optional; |
|
| 12 |
|
|
| 13 |
@Component |
|
| 14 |
public class ResearchProductCsvConverter {
|
|
| 15 |
|
|
| 16 |
@Autowired |
|
| 17 |
ResearchProductCsvMapper researchProductCsvMapper; |
|
| 18 |
|
|
| 19 |
public List<ResearchProductCsv> flattenProjects(SolrRecord solrRecord) {
|
|
| 20 |
List<ResearchProductCsv> flattenedList = new ArrayList<>(); |
|
| 21 |
|
|
| 22 |
if (solrRecord.getLinks() != null) {
|
|
| 23 |
boolean hasProducedRecord = false; |
|
| 24 |
|
|
| 25 |
for (RelatedRecord relatedRecord : solrRecord.getLinks()) {
|
|
| 26 |
if (isProducedByRelation(relatedRecord)) {
|
|
| 27 |
ResearchProductCsv newProduct = mapToResearchProductWithFunder(solrRecord, relatedRecord); |
|
| 28 |
flattenedList.add(newProduct); |
|
| 29 |
hasProducedRecord = true; |
|
| 30 |
} |
|
| 31 |
} |
|
| 32 |
|
|
| 33 |
// If no "isProducedBy" records found, map the entire solrRecord |
|
| 34 |
if (!hasProducedRecord) {
|
|
| 35 |
ResearchProductCsv defaultProduct = researchProductCsvMapper.toResearchProductCsv(solrRecord); |
|
| 36 |
flattenedList.add(defaultProduct); |
|
| 37 |
} |
|
| 38 |
} |
|
| 39 |
|
|
| 40 |
return flattenedList; |
|
| 41 |
} |
|
| 42 |
|
|
| 43 |
private boolean isProducedByRelation(RelatedRecord relatedRecord) {
|
|
| 44 |
return relatedRecord.getHeader() != null |
|
| 45 |
&& "isProducedBy".equalsIgnoreCase(relatedRecord.getHeader().getRelationClass()); |
|
| 46 |
} |
|
| 47 |
|
|
| 48 |
private ResearchProductCsv mapToResearchProductWithFunder(SolrRecord solrRecord, RelatedRecord relatedRecord) {
|
|
| 49 |
ResearchProductCsv newProduct = researchProductCsvMapper.toSimpleResearchProductCsv(solrRecord); |
|
| 50 |
|
|
| 51 |
String funderString = Optional.ofNullable(relatedRecord.getFunding()) |
|
| 52 |
.map(funding -> Optional.ofNullable(funding.getFunder()) |
|
| 53 |
.map(funder -> Optional.ofNullable(funder.getShortname()) |
|
| 54 |
.orElse(funder.getName()) |
|
| 55 |
).orElse("")
|
|
| 56 |
).orElse("");
|
|
| 57 |
|
|
| 58 |
String projectString = Optional.ofNullable(relatedRecord.getAcronym()) |
|
| 59 |
.orElseGet(() -> Optional.ofNullable(relatedRecord.getProjectTitle()) |
|
| 60 |
.orElse("Unknown"));
|
|
| 61 |
|
|
| 62 |
String funderProject = funderString + "|" + projectString; |
|
| 63 |
|
|
| 64 |
String code = Optional.ofNullable(relatedRecord.getCode()) |
|
| 65 |
.filter(c -> !"unidentified".equalsIgnoreCase(c)) |
|
| 66 |
.map(c -> "(" + c + ")")
|
|
| 67 |
.orElse("");
|
|
| 68 |
|
|
| 69 |
newProduct.setFunder_Project(funderProject + code); |
|
| 70 |
|
|
| 71 |
return newProduct; |
|
| 72 |
} |
|
| 73 |
} |
|
| modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/helpers/html/HtmlEntity.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.search.helpers.html; |
|
| 2 |
|
|
| 3 |
public interface HtmlEntity {
|
|
| 4 |
} |
|
| modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/helpers/html/ResearchProductHtml.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.search.helpers.html; |
|
| 2 |
|
|
| 3 |
import lombok.Data; |
|
| 4 |
|
|
| 5 |
@Data |
|
| 6 |
public class ResearchProductHtml implements HtmlEntity {
|
|
| 7 |
String type; |
|
| 8 |
String title; |
|
| 9 |
String authors; |
|
| 10 |
String publicationYear; |
|
| 11 |
String doi; |
|
| 12 |
String permanentIdentifier; |
|
| 13 |
String publicationType; |
|
| 14 |
String journal; |
|
| 15 |
String projectName_gaNumber; |
|
| 16 |
String accessMode; |
|
| 17 |
} |
|
| modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/mappers/csv/DatasourceCsvMapper.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.search.mappers.csv; |
|
| 2 |
|
|
| 3 |
import eu.dnetlib.data.search.helpers.csv.DatasourceCsv; |
|
| 4 |
import eu.dnetlib.dhp.schema.solr.SolrRecord; |
|
| 5 |
import org.mapstruct.Mapper; |
|
| 6 |
import org.mapstruct.Mapping; |
|
| 7 |
|
|
| 8 |
@Mapper(componentModel = "spring") |
|
| 9 |
public interface DatasourceCsvMapper {
|
|
| 10 |
|
|
| 11 |
@Mapping(target="name", source = "datasource.officialname") |
|
| 12 |
@Mapping(target="type", source = "datasource.datasourcetype.label") |
|
| 13 |
@Mapping(target="compatibility", source = "datasource.openairecompatibility.label") |
|
| 14 |
@Mapping(target = "oaipmh", source = "datasource.accessinfopackage") |
|
| 15 |
@Mapping(target="websiteUrl", source="datasource.websiteurl") |
|
| 16 |
DatasourceCsv toDatasourceCsv(SolrRecord solrRecord); |
|
| 17 |
} |
|
| modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/mappers/csv/OrganizationCsvMapper.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.search.mappers.csv; |
|
| 2 |
|
|
| 3 |
import eu.dnetlib.data.search.helpers.csv.OrganizationCsv; |
|
| 4 |
import eu.dnetlib.dhp.schema.solr.SolrRecord; |
|
| 5 |
import org.mapstruct.Mapper; |
|
| 6 |
import org.mapstruct.Mapping; |
|
| 7 |
|
|
| 8 |
@Mapper(componentModel = "spring") |
|
| 9 |
public interface OrganizationCsvMapper {
|
|
| 10 |
|
|
| 11 |
@Mapping(target="name", source = "organization.legalname") |
|
| 12 |
@Mapping(target="country", source="organization.country.label") |
|
| 13 |
OrganizationCsv toOrganizationCsv(SolrRecord solrRecord); |
|
| 14 |
} |
|
| modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/mappers/csv/ProjectCsvMapper.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.search.mappers.csv; |
|
| 2 |
|
|
| 3 |
import eu.dnetlib.data.search.helpers.csv.ProjectCsv; |
|
| 4 |
import eu.dnetlib.dhp.schema.solr.Funding; |
|
| 5 |
import eu.dnetlib.dhp.schema.solr.Project; |
|
| 6 |
import eu.dnetlib.dhp.schema.solr.RelatedRecord; |
|
| 7 |
import eu.dnetlib.dhp.schema.solr.SolrRecord; |
|
| 8 |
import org.mapstruct.Mapper; |
|
| 9 |
import org.mapstruct.Mapping; |
|
| 10 |
|
|
| 11 |
import java.util.List; |
|
| 12 |
import java.util.Optional; |
|
| 13 |
import java.util.stream.Collectors; |
|
| 14 |
import java.util.stream.Stream; |
|
| 15 |
|
|
| 16 |
@Mapper(componentModel = "spring") |
|
| 17 |
public interface ProjectCsvMapper {
|
|
| 18 |
|
|
| 19 |
@Mapping(target="title", source="project.title") |
|
| 20 |
@Mapping(target="funder", expression="java(extractFunder(solrRecord))") |
|
| 21 |
@Mapping(target="startDate", source="project.startdate") |
|
| 22 |
@Mapping(target="endDate", source="project.enddate") |
|
| 23 |
@Mapping(target="participants", expression="java(extractParticipants(solrRecord))") |
|
| 24 |
ProjectCsv toProjectCsv(SolrRecord solrRecord); |
|
| 25 |
|
|
| 26 |
default String extractFunder(SolrRecord solrRecord) {
|
|
| 27 |
return Optional.ofNullable(solrRecord) |
|
| 28 |
.map(SolrRecord::getProject) |
|
| 29 |
.map(Project::getFunding) |
|
| 30 |
.map(Funding::getFunder) |
|
| 31 |
.map(funder ->{
|
|
| 32 |
String funderName = Optional.ofNullable(funder.getName()).orElse("");
|
|
| 33 |
String funderShortName = Optional.ofNullable(funder.getShortname()).orElse("");
|
|
| 34 |
|
|
| 35 |
return funderShortName.isEmpty() |
|
| 36 |
? funderName |
|
| 37 |
: funderName + " (" + funderShortName + ")";
|
|
| 38 |
}).orElse("");
|
|
| 39 |
} |
|
| 40 |
|
|
| 41 |
default List<String> extractParticipants(SolrRecord solrRecord) {
|
|
| 42 |
return Optional.ofNullable(solrRecord.getLinks()) |
|
| 43 |
.map(List::stream) |
|
| 44 |
.orElseGet(Stream::empty) |
|
| 45 |
.filter(relatedRecord -> "hasParticipant".equals(relatedRecord.getHeader().getRelationClass())) |
|
| 46 |
.map(RelatedRecord::getLegalname) |
|
| 47 |
.filter(legalName -> legalName != null) |
|
| 48 |
.collect(Collectors.toList()); |
|
| 49 |
} |
|
| 50 |
} |
|
| modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/mappers/csv/ResearchProductCsvMapper.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.search.mappers.csv; |
|
| 2 |
|
|
| 3 |
import eu.dnetlib.data.search.helpers.csv.ResearchProductCsv; |
|
| 4 |
import eu.dnetlib.dhp.schema.solr.Author; |
|
| 5 |
import eu.dnetlib.dhp.schema.solr.Pid; |
|
| 6 |
import eu.dnetlib.dhp.schema.solr.SolrRecord; |
|
| 7 |
import org.mapstruct.Mapper; |
|
| 8 |
import org.mapstruct.Mapping; |
|
| 9 |
|
|
| 10 |
import java.util.Collection; |
|
| 11 |
import java.util.List; |
|
| 12 |
import java.util.Objects; |
|
| 13 |
import java.util.Optional; |
|
| 14 |
import java.util.stream.Collectors; |
|
| 15 |
import java.util.stream.Stream; |
|
| 16 |
|
|
| 17 |
@Mapper(componentModel = "spring") |
|
| 18 |
public interface ResearchProductCsvMapper {
|
|
| 19 |
|
|
| 20 |
@Mapping(target="researchProduct", source = "result.resulttype") |
|
| 21 |
@Mapping(target="title", source="result.maintitle") |
|
| 22 |
@Mapping(target="authors", expression = "java(extractAuthor(solrRecord))") |
|
| 23 |
@Mapping(target = "publicationYear", source = "result.publicationdate") |
|
| 24 |
@Mapping(target = "doi", expression = "java(extractDoi(solrRecord))") |
|
| 25 |
@Mapping(target = "type", source = "result.resulttype") |
|
| 26 |
@Mapping(target = "peerReviewed", expression = "java(extractPeerReviewed(solrRecord))") |
|
| 27 |
@Mapping(target = "diamondJournal", source = "result.inDiamondJournal") |
|
| 28 |
@Mapping(target = "journal", source = "result.journal.name") |
|
| 29 |
@Mapping(target = "downloadFrom", expression = "java(extractDownloadFrom(solrRecord))") |
|
| 30 |
@Mapping(target = "publiclyFunded", source = "result.publiclyFunded") |
|
| 31 |
@Mapping(target = "funder_Project", expression = "java(extractFunder(solrRecord))") |
|
| 32 |
@Mapping(target = "access", source = "result.bestaccessright.label") |
|
| 33 |
@Mapping(target = "green", source = "result.green") |
|
| 34 |
@Mapping(target = "oaColour", source = "result.openAccessColor") |
|
| 35 |
ResearchProductCsv toResearchProductCsv(SolrRecord solrRecord); |
|
| 36 |
|
|
| 37 |
@Mapping(target="researchProduct", source = "result.resulttype") |
|
| 38 |
@Mapping(target="title", source="result.maintitle") |
|
| 39 |
@Mapping(target="authors", expression = "java(extractAuthor(solrRecord))") |
|
| 40 |
@Mapping(target = "publicationYear", source = "result.publicationdate") |
|
| 41 |
@Mapping(target = "doi", expression = "java(extractDoi(solrRecord))") |
|
| 42 |
@Mapping(target = "type", source = "result.resulttype") |
|
| 43 |
@Mapping(target = "peerReviewed", expression = "java(extractPeerReviewed(solrRecord))") |
|
| 44 |
@Mapping(target = "diamondJournal", source = "result.inDiamondJournal") |
|
| 45 |
@Mapping(target = "journal", source = "result.journal.name") |
|
| 46 |
@Mapping(target = "downloadFrom", expression = "java(extractDownloadFrom(solrRecord))") |
|
| 47 |
@Mapping(target = "publiclyFunded", source = "result.publiclyFunded") |
|
| 48 |
@Mapping(target = "access", source = "result.bestaccessright.label") |
|
| 49 |
@Mapping(target = "green", source = "result.green") |
|
| 50 |
@Mapping(target = "oaColour", source = "result.openAccessColor") |
|
| 51 |
ResearchProductCsv toSimpleResearchProductCsv(SolrRecord solrRecord); |
|
| 52 |
|
|
| 53 |
default List<String> extractAuthor(SolrRecord solrRecord) {
|
|
| 54 |
return Optional.ofNullable(solrRecord.getResult().getAuthor()) |
|
| 55 |
.map(Collection::stream) |
|
| 56 |
.orElseGet(Stream::empty) |
|
| 57 |
.map(Author::getFullname) |
|
| 58 |
.filter(Objects::nonNull) |
|
| 59 |
.collect(Collectors.toList()); |
|
| 60 |
} |
|
| 61 |
|
|
| 62 |
default String extractFunder(SolrRecord solrRecord) {
|
|
| 63 |
return Optional.ofNullable(solrRecord.getLinks()) |
|
| 64 |
.map(List::stream) |
|
| 65 |
.orElseGet(Stream::empty) |
|
| 66 |
.filter(relatedRecord -> "isProducedBy".equalsIgnoreCase(relatedRecord.getHeader().getRelationClass())) |
|
| 67 |
.map(relatedRecord -> {
|
|
| 68 |
//System.out.println("Funded!!!!!!!");
|
|
| 69 |
String funderString = Optional.ofNullable(relatedRecord.getFunding()) |
|
| 70 |
.map(funding -> Optional.ofNullable(funding.getFunder()) |
|
| 71 |
.map(funder -> Optional.ofNullable(funder.getShortname()) |
|
| 72 |
.orElse(funder.getName()) |
|
| 73 |
).orElse("")
|
|
| 74 |
).orElse("");
|
|
| 75 |
|
|
| 76 |
String projectString = Optional.ofNullable(relatedRecord.getAcronym()) |
|
| 77 |
.orElseGet(() -> Optional.ofNullable(relatedRecord.getProjectTitle()) |
|
| 78 |
.orElse("Unknown"));
|
|
| 79 |
|
|
| 80 |
funderString += "|" + projectString; |
|
| 81 |
|
|
| 82 |
String code = Optional.ofNullable(relatedRecord.getCode()) |
|
| 83 |
.map(c -> "(" + c + ")")
|
|
| 84 |
.orElse("");
|
|
| 85 |
|
|
| 86 |
return funderString + code; |
|
| 87 |
}) |
|
| 88 |
.collect(Collectors.joining(";"));
|
|
| 89 |
} |
|
| 90 |
|
|
| 91 |
default List<String> extractDoi(SolrRecord solrRecord) {
|
|
| 92 |
return Optional.ofNullable(solrRecord.getPid()) |
|
| 93 |
.map(List::stream) |
|
| 94 |
.orElseGet(Stream::empty) |
|
| 95 |
.filter(pid -> "doi".equalsIgnoreCase(pid.getType())) |
|
| 96 |
.map(Pid::getValue) |
|
| 97 |
.collect(Collectors.toList()); |
|
| 98 |
} |
|
| 99 |
|
|
| 100 |
default String extractPeerReviewed(SolrRecord solrRecord){
|
|
| 101 |
return Optional.ofNullable(solrRecord.getResult().getInstance()) |
|
| 102 |
.map(List::stream) |
|
| 103 |
.orElseGet(Stream::empty) |
|
| 104 |
.filter(instance -> "peerReviewed".equalsIgnoreCase(instance.getRefereed())) |
|
| 105 |
.findFirst() |
|
| 106 |
.map(instance -> "yes") |
|
| 107 |
.orElse("no");
|
|
| 108 |
} |
|
| 109 |
|
|
| 110 |
default String extractDownloadFrom(SolrRecord solrRecord) {
|
|
| 111 |
return Optional.ofNullable(solrRecord.getResult().getInstance()) |
|
| 112 |
.map(List::stream) // Convert the instance list to a stream if it's not null |
|
| 113 |
.orElseGet(Stream::empty) // Provide an empty stream if the instance is null |
|
| 114 |
.filter(instance -> "OPEN".equalsIgnoreCase(instance.getAccessright().getValue())) // Filter for "OPEN" |
|
| 115 |
.findFirst() // Find the first matching instance |
|
| 116 |
.map(instance -> |
|
| 117 |
Optional.ofNullable(instance.getUrl()) // Get the list of URLs |
|
| 118 |
.filter(urls -> !urls.isEmpty()) // Ensure the list is not empty |
|
| 119 |
.map(urls -> urls.get(0)) // Get the first URL from the list |
|
| 120 |
.orElse("")) // Return empty string if the list is empty
|
|
| 121 |
.orElseGet(() -> {
|
|
| 122 |
// Handle case if the first part didn't find a matching instance |
|
| 123 |
return Optional.ofNullable(solrRecord.getResult().getInstance()) |
|
| 124 |
.map(List::stream) // Convert the instance list to a stream if it's not null |
|
| 125 |
.orElseGet(Stream::empty) // Provide an empty stream if the instance is null |
|
| 126 |
.findFirst() // Find the first instance |
|
| 127 |
.map(instance -> |
|
| 128 |
Optional.ofNullable(instance.getUrl()) // Get the list of URLs |
|
| 129 |
.filter(urls -> !urls.isEmpty()) // Ensure the list is not empty |
|
| 130 |
.map(urls -> urls.get(0)) // Get the first URL from the list |
|
| 131 |
.orElse("")) // Return empty string if the list is empty
|
|
| 132 |
.orElse(""); // Return empty string if no instance is found
|
|
| 133 |
}); |
|
| 134 |
} |
|
| 135 |
} |
|
| modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/mappers/html/ResearchProductsHtmlMapper.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.search.mappers.html; |
|
| 2 |
|
|
| 3 |
import eu.dnetlib.data.search.helpers.html.ResearchProductHtml; |
|
| 4 |
import eu.dnetlib.dhp.schema.solr.Author; |
|
| 5 |
import eu.dnetlib.dhp.schema.solr.Pid; |
|
| 6 |
import eu.dnetlib.dhp.schema.solr.SolrRecord; |
|
| 7 |
import org.mapstruct.Mapper; |
|
| 8 |
import org.mapstruct.Mapping; |
|
| 9 |
|
|
| 10 |
import java.util.Collection; |
|
| 11 |
import java.util.List; |
|
| 12 |
import java.util.Objects; |
|
| 13 |
import java.util.Optional; |
|
| 14 |
import java.util.stream.Collectors; |
|
| 15 |
import java.util.stream.Stream; |
|
| 16 |
|
|
| 17 |
@Mapper(componentModel = "spring") |
|
| 18 |
public interface ResearchProductsHtmlMapper {
|
|
| 19 |
@Mapping(target = "type", source = "result.resulttype") |
|
| 20 |
@Mapping(target = "title", source = "result.maintitle") |
|
| 21 |
@Mapping(target = "authors", expression = "java(extractAuthor(solrRecord))") |
|
| 22 |
@Mapping(target = "publicationYear", source = "result.publicationdate") |
|
| 23 |
@Mapping(target = "doi", expression = "java(extractDoi(solrRecord))") |
|
| 24 |
@Mapping(target = "permanentIdentifier", expression = "java(extractPermanentIdentifier(solrRecord))") |
|
| 25 |
@Mapping(target = "publicationType", expression = "java(exctractPublicationType(solrRecord))") |
|
| 26 |
@Mapping(target = "journal", source = "result.journal.name") |
|
| 27 |
@Mapping(target = "projectName_gaNumber", expression = "java(extractFunder(solrRecord))") |
|
| 28 |
@Mapping(target = "accessMode", source = "result.bestaccessright.label") |
|
| 29 |
ResearchProductHtml toResearchProductHtml(SolrRecord solrRecord); |
|
| 30 |
|
|
| 31 |
default String extractAuthor(SolrRecord solrRecord) {
|
|
| 32 |
return Optional.ofNullable(solrRecord.getResult().getAuthor()) |
|
| 33 |
.map(authors -> authors.stream() |
|
| 34 |
.map(Author::getFullname) |
|
| 35 |
.filter(Objects::nonNull) |
|
| 36 |
.collect(Collectors.joining(";")))
|
|
| 37 |
.orElse("");
|
|
| 38 |
} |
|
| 39 |
|
|
| 40 |
default String extractPermanentIdentifier(SolrRecord solrRecord){
|
|
| 41 |
return Optional.ofNullable(solrRecord.getResult().getInstance()) |
|
| 42 |
.map(instances -> instances.stream() |
|
| 43 |
.findFirst() |
|
| 44 |
.map(instance -> Optional.ofNullable(instance.getUrl()) |
|
| 45 |
.filter(urls -> !urls.isEmpty()) |
|
| 46 |
.map(urls -> urls.get(0)).orElse(""))
|
|
| 47 |
.orElse("")).toString();
|
|
| 48 |
} |
|
| 49 |
|
|
| 50 |
default String exctractPublicationType(SolrRecord solrRecord){
|
|
| 51 |
return Optional.ofNullable(solrRecord.getResult().getInstance()) |
|
| 52 |
.map(instances -> instances.stream() |
|
| 53 |
.findFirst() |
|
| 54 |
.map(instance -> instance.getInstancetype()).orElse(""))
|
|
| 55 |
.orElse("");
|
|
| 56 |
} |
|
| 57 |
|
|
| 58 |
default String extractDoi(SolrRecord solrRecord) {
|
|
| 59 |
return Optional.ofNullable(solrRecord.getPid()) |
|
| 60 |
.map(pids -> pids.stream() |
|
| 61 |
.filter(pid -> "doi".equalsIgnoreCase(pid.getType())) |
|
| 62 |
.map(Pid::getValue) |
|
| 63 |
.collect(Collectors.joining(";")))
|
|
| 64 |
.orElse("");
|
|
| 65 |
} |
|
| 66 |
|
|
| 67 |
default String extractFunder(SolrRecord solrRecord) {
|
|
| 68 |
return Optional.ofNullable(solrRecord.getLinks()) |
|
| 69 |
.map(List::stream) |
|
| 70 |
.orElseGet(Stream::empty) |
|
| 71 |
.filter(relatedRecord -> "isProducedBy".equalsIgnoreCase(relatedRecord.getHeader().getRelationClass())) |
|
| 72 |
.map(relatedRecord -> {
|
|
| 73 |
//System.out.println("Funded!!!!!!!");
|
|
| 74 |
String funderString = Optional.ofNullable(relatedRecord.getFunding()) |
|
| 75 |
.map(funding -> Optional.ofNullable(funding.getFunder()) |
|
| 76 |
.map(funder -> Optional.ofNullable(funder.getShortname()) |
|
| 77 |
.orElse(funder.getName()) |
|
| 78 |
).orElse("")
|
|
| 79 |
).orElse("");
|
|
| 80 |
|
|
| 81 |
String projectString = Optional.ofNullable(relatedRecord.getAcronym()) |
|
| 82 |
.orElseGet(() -> Optional.ofNullable(relatedRecord.getProjectTitle()) |
|
| 83 |
.orElse("Unknown"));
|
|
| 84 |
|
|
| 85 |
funderString += "|" + projectString; |
|
| 86 |
|
|
| 87 |
String code = Optional.ofNullable(relatedRecord.getCode()) |
|
| 88 |
.map(c -> "(" + c + ")")
|
|
| 89 |
.orElse("");
|
|
| 90 |
|
|
| 91 |
return funderString + code; |
|
| 92 |
}) |
|
| 93 |
.collect(Collectors.joining(";"));
|
|
| 94 |
} |
|
| 95 |
} |
|
| modules/uoa-search/branches/uoa-search-json/src/main/java/eu/dnetlib/data/search/solr/SolrResultSet.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.data.search.solr; |
| 2 | 2 |
|
| 3 |
import com.fasterxml.jackson.core.type.TypeReference; |
|
| 4 |
import com.fasterxml.jackson.databind.DeserializationFeature; |
|
| 5 |
import com.fasterxml.jackson.databind.ObjectMapper; |
|
| 6 |
import com.fasterxml.jackson.databind.ObjectWriter; |
|
| 7 |
import com.fasterxml.jackson.dataformat.csv.CsvMapper; |
|
| 8 |
import com.fasterxml.jackson.dataformat.csv.CsvSchema; |
|
| 3 | 9 |
import com.google.gson.Gson; |
| 4 | 10 |
import eu.dnetlib.api.data.SearchServiceException; |
| 11 |
import eu.dnetlib.data.search.helpers.csv.*; |
|
| 12 |
import eu.dnetlib.data.search.helpers.html.ResearchProductHtml; |
|
| 13 |
import eu.dnetlib.data.search.mappers.csv.DatasourceCsvMapper; |
|
| 14 |
import eu.dnetlib.data.search.mappers.csv.OrganizationCsvMapper; |
|
| 15 |
import eu.dnetlib.data.search.mappers.csv.ProjectCsvMapper; |
|
| 16 |
import eu.dnetlib.data.search.mappers.csv.ResearchProductCsvMapper; |
|
| 17 |
import eu.dnetlib.data.search.mappers.html.ResearchProductsHtmlMapper; |
|
| 5 | 18 |
import eu.dnetlib.data.search.transform.Transformer; |
| 6 | 19 |
import eu.dnetlib.data.search.transform.TransformerException; |
| 7 | 20 |
import eu.dnetlib.data.search.utils.solr.SolrResultSetOptionsUtil; |
| 8 | 21 |
import eu.dnetlib.data.search.utils.solr.SolrResultsFormatter; |
| 22 |
import eu.dnetlib.dhp.schema.solr.*; |
|
| 9 | 23 |
import eu.dnetlib.domain.EPR; |
| 10 | 24 |
import gr.uoa.di.driver.enabling.resultset.ResultSet; |
| 11 | 25 |
import io.micrometer.core.instrument.Timer; |
| ... | ... | |
| 20 | 34 |
import org.apache.solr.common.SolrDocumentList; |
| 21 | 35 |
import org.apache.solr.common.params.SolrParams; |
| 22 | 36 |
import org.apache.solr.common.util.NamedList; |
| 37 |
import org.springframework.beans.factory.annotation.Autowired; |
|
| 23 | 38 |
import org.z3950.zing.cql.CQLParseException; |
| 24 | 39 |
|
| 25 | 40 |
import javax.ws.rs.core.MediaType; |
| ... | ... | |
| 41 | 56 |
private NamedList<String> queryOpts = new NamedList<String>(); |
| 42 | 57 |
long size = -1; |
| 43 | 58 |
|
| 59 |
final ObjectMapper objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); |
|
| 60 |
|
|
| 44 | 61 |
private PrometheusMeterRegistry registry; |
| 45 | 62 |
|
| 46 |
public SolrResultSet(EPR epr, CloudSolrClient cloudSolrClient, PrometheusMeterRegistry registry) |
|
| 63 |
DatasourceCsvMapper datasourceCsvMapper; |
|
| 64 |
OrganizationCsvMapper organizationCsvMapper; |
|
| 65 |
ProjectCsvMapper projectCsvMapper; |
|
| 66 |
ResearchProductCsvMapper researchProductCsvMapper; |
|
| 67 |
ResearchProductCsvConverter researchProductCsvConverter; |
|
| 68 |
ResearchProductsHtmlMapper researchProductsHtmlMapper; |
|
| 69 |
|
|
| 70 |
CsvMapper datasourceMapper = new CsvMapper(); |
|
| 71 |
CsvMapper organizationMapper = new CsvMapper(); |
|
| 72 |
CsvMapper projectMapper = new CsvMapper(); |
|
| 73 |
CsvMapper researchProductMapper = new CsvMapper(); |
|
| 74 |
|
|
| 75 |
public SolrResultSet(EPR epr, CloudSolrClient cloudSolrClient, PrometheusMeterRegistry registry, DatasourceCsvMapper |
|
| 76 |
datasourceCsvMapper, OrganizationCsvMapper organizationCsvMapper, ProjectCsvMapper projectCsvMapper, |
|
| 77 |
ResearchProductCsvMapper researchProductCsvMapper, ResearchProductCsvConverter researchProductCsvConverter, |
|
| 78 |
ResearchProductsHtmlMapper researchProductsHtmlMapper) |
|
| 47 | 79 |
throws IOException, CQLParseException, SearchServiceException {
|
| 48 | 80 |
this.epr = epr; |
| 49 | 81 |
this.solrClient = cloudSolrClient; |
| ... | ... | |
| 54 | 86 |
String mdformat = epr.getParameter("mdformat");
|
| 55 | 87 |
String interpretation = epr.getParameter("interpretation");
|
| 56 | 88 |
|
| 89 |
this.datasourceCsvMapper = datasourceCsvMapper; |
|
| 90 |
this.organizationCsvMapper = organizationCsvMapper; |
|
| 91 |
this.projectCsvMapper = projectCsvMapper; |
|
| 92 |
this.researchProductCsvMapper = researchProductCsvMapper; |
|
| 93 |
this.researchProductCsvConverter = researchProductCsvConverter; |
|
| 94 |
this.researchProductsHtmlMapper = researchProductsHtmlMapper; |
|
| 95 |
|
|
| 57 | 96 |
((CloudSolrClient)solrClient).setDefaultCollection(mdformat + "-" + layout + "-" + interpretation); |
| 58 | 97 |
|
| 59 | 98 |
this.registry = registry; |
| ... | ... | |
| 170 | 209 |
} |
| 171 | 210 |
|
| 172 | 211 |
public Map<String,List<String>> newGet(int from, int to, String format, Transformer transformer, |
| 173 |
Transformer oldRefineTransformer, boolean timeout, String timeoutValue) throws SearchServiceException {
|
|
| 212 |
Transformer oldRefineTransformer, boolean timeout, String timeoutValue) throws SearchServiceException {
|
|
| 174 | 213 |
List<String> refineSolrResults = new ArrayList<String>(); |
| 175 | 214 |
List<String> searchSolrResults = new ArrayList<String>(); |
| 176 | 215 |
|
| ... | ... | |
| 334 | 373 |
} |
| 335 | 374 |
} |
| 336 | 375 |
|
| 376 |
|
|
| 377 |
public Map<String,List<String>> newJsonGet(int from, int to, String format, Transformer transformer, |
|
| 378 |
Transformer oldRefineTransformer, boolean timeout, String timeoutValue) throws SearchServiceException {
|
|
| 379 |
List<String> refineSolrResults = new ArrayList<String>(); |
|
| 380 |
List<String> searchSolrResults = new ArrayList<String>(); |
|
| 381 |
|
|
| 382 |
QueryResponse rsp = null; |
|
| 383 |
HashMap<String, List<String>> map = new HashMap<String, List<String>>(); |
|
| 384 |
|
|
| 385 |
if (from < 0) from = 0; |
|
| 386 |
if (to < 0) to = 0; |
|
| 387 |
queryOpts.add("start", from*to + "");
|
|
| 388 |
queryOpts.add("rows", to +"");
|
|
| 389 |
|
|
| 390 |
try {
|
|
| 391 |
if (timeout) {
|
|
| 392 |
if (timeoutValue != null && Integer.valueOf(timeoutValue) == -1) {
|
|
| 393 |
logger.warn("Timeout value is -1. Timeout will not be set for query: " + queryOpts.get("q"));
|
|
| 394 |
timeout = false; |
|
| 395 |
} else if (Integer.valueOf(timeoutValue) < 500) {
|
|
| 396 |
logger.warn("Timeout is less than 500ms. We fall back to the default 1000ms for query:" + queryOpts.get("q"));
|
|
| 397 |
queryOpts.add("timeAllowed", "500");
|
|
| 398 |
} else if (timeout) {
|
|
| 399 |
queryOpts.add("timeAllowed", timeoutValue);
|
|
| 400 |
} |
|
| 401 |
} |
|
| 402 |
} catch (NumberFormatException nfe) {
|
|
| 403 |
logger.warn("NumberFormatException occurred. Timeout not properly set. We fall back to the default 1000ms.");
|
|
| 404 |
queryOpts.add("timeAllowed", "500");
|
|
| 405 |
} |
|
| 406 |
|
|
| 407 |
logger.info("timeout: " + timeout + ", timeout value: " + queryOpts.get("timeAllowed"));
|
|
| 408 |
|
|
| 409 |
//queryOpts.add("f.resulthostingdatasource.facet.limit", "2");
|
|
| 410 |
|
|
| 411 |
long startTime = System.nanoTime(); |
|
| 412 |
|
|
| 413 |
try {
|
|
| 414 |
io.micrometer.core.instrument.Timer.Sample timer = Timer.start(registry); |
|
| 415 |
rsp = solrClient.query(SolrParams.toSolrParams(queryOpts)); |
|
| 416 |
timer.stop(registry.timer("solr.server.response.duration"));
|
|
| 417 |
|
|
| 418 |
long estimatedTime = System.nanoTime() - startTime; |
|
| 419 |
logger.info("Solrj time " + estimatedTime/1000000 + " milliseconds for query:" + queryOpts.get("q") +
|
|
| 420 |
" and facets " + queryOpts.getAll("facet.field") + " and fq " + queryOpts.getAll("fq") +
|
|
| 421 |
" and sorts " + queryOpts.getAll("sort") + " from: "
|
|
| 422 |
+ from + " and size " + to + " with timeout: " + timeout); |
|
| 423 |
|
|
| 424 |
/*System.out.println("Solrj time " + estimatedTime/1000000 + " milliseconds for query:" + queryOpts.get("q") +
|
|
| 425 |
" and facets " + queryOpts.getAll("facet.field") + " and fq " + queryOpts.getAll("fq") +
|
|
| 426 |
" and sorts " + queryOpts.getAll("sort") + " from: "
|
|
| 427 |
+ from + " and size " + to); |
|
| 428 |
*/ |
|
| 429 |
facetFields = rsp.getFacetFields(); |
|
| 430 |
|
|
| 431 |
SolrDocumentList docs = rsp.getResults(); |
|
| 432 |
|
|
| 433 |
this.size = docs.getNumFound(); |
|
| 434 |
|
|
| 435 |
if (facetFields!=null && !facetFields.isEmpty()) {
|
|
| 436 |
System.out.println("111111");
|
|
| 437 |
if (format != null && format.equals(MediaType.APPLICATION_JSON)) {
|
|
| 438 |
System.out.println("22222");
|
|
| 439 |
for (FacetField field : facetFields) {
|
|
| 440 |
System.out.println("33333 " + field.getValueCount());
|
|
| 441 |
map.put(field.getName(), new ArrayList<String>()); |
|
| 442 |
BrowseField bf = null; |
|
| 443 |
for (int i = 0; i < field.getValueCount(); i++) {
|
|
| 444 |
System.out.println("44444");
|
|
| 445 |
bf = new BrowseField(); |
|
| 446 |
//bf.setId(org.apache.commons.lang3.StringEscapeUtils.escapeJson(field.getValues().get(i).getName())); |
|
| 447 |
bf.setId(field.getValues().get(i).getName()); |
|
| 448 |
String[] facetedValues = field.getValues().get(i).getName().split("\\|\\|",2);
|
|
| 449 |
|
|
| 450 |
|
|
| 451 |
logger.debug("faceted values " + Arrays.toString(facetedValues));
|
|
| 452 |
System.out.println("faceted values " + Arrays.toString(facetedValues));
|
|
| 453 |
|
|
| 454 |
if (facetedValues.length > 1) {
|
|
| 455 |
//bf.setName(org.apache.commons.lang3.StringEscapeUtils.escapeJson(facetedValues[1])); |
|
| 456 |
bf.setName(facetedValues[1]); |
|
| 457 |
logger.debug("faceted values [1] " + facetedValues[1]);
|
|
| 458 |
|
|
| 459 |
} else if (field.getValues().get(i).getName().split("_\\:\\:",2).length > 1) {
|
|
| 460 |
//bf.setName(org.apache.commons.lang3.StringEscapeUtils.escapeJson(field.getValues().get(i).getName().split("\\:\\:",2)[1]).replaceAll("\\:\\:", "\\|"));
|
|
| 461 |
bf.setName(field.getValues().get(i).getName().split("\\:\\:",2)[1].replaceAll("\\:\\:", "\\|"));
|
|
| 462 |
|
|
| 463 |
} else {
|
|
| 464 |
//bf.setName(org.apache.commons.lang3.StringEscapeUtils.escapeJson(field.getValues().get(i).getName())); |
|
| 465 |
bf.setName(field.getValues().get(i).getName()); |
|
| 466 |
} |
|
| 467 |
|
|
| 468 |
bf.setCount(field.getValues().get(i).getCount() + ""); |
|
| 469 |
System.out.println("skatakia:>:>" + new Gson().toJson(bf));
|
|
| 470 |
map.get(field.getName()).add(new Gson().toJson(bf)); |
|
| 471 |
} |
|
| 472 |
|
|
| 473 |
} |
|
| 474 |
|
|
| 475 |
StringBuilder builder = null; |
|
| 476 |
|
|
| 477 |
for (Map.Entry<String, List<String>> facetEntry : map.entrySet()) {
|
|
| 478 |
builder = new StringBuilder(); |
|
| 479 |
builder.append("\"" + facetEntry.getKey() + "\"" + " : ");
|
|
| 480 |
builder.append(facetEntry.getValue()); |
|
| 481 |
refineSolrResults.add(builder.toString()); |
|
| 482 |
} |
|
| 483 |
|
|
| 484 |
} else { //the old implementation & xml as default
|
|
| 485 |
logger.debug("Creating old browse results.");
|
|
| 486 |
createXmlRefineFields(refineSolrResults, oldRefineTransformer); |
|
| 487 |
} |
|
| 488 |
} |
|
| 489 |
|
|
| 490 |
startTime = System.nanoTime(); |
|
| 491 |
|
|
| 492 |
for (int i = 0; i < docs.size(); i++) {
|
|
| 493 |
String result = (String) docs.get(i).get("__json");
|
|
| 494 |
|
|
| 495 |
//System.out.println("\n\n\n\n\n JSON ---------------\n"+ result);
|
|
| 496 |
//logger.debug("["+ i +"]: " + docs.get(i).get("__result"));
|
|
| 497 |
|
|
| 498 |
/*try {
|
|
| 499 |
if (transformer != null) {
|
|
| 500 |
//logger.debug("1 >>>>>>" + result);
|
|
| 501 |
String xml = result.replaceAll("<em>","").replaceAll("</em>","");
|
|
| 502 |
result = transformer.transform(xml); |
|
| 503 |
//logger.debug("2 >>>>>>" + result);
|
|
| 504 |
} |
|
| 505 |
|
|
| 506 |
} catch (TransformerException te) {
|
|
| 507 |
logger.warn("Error transforming " + result, te);
|
|
| 508 |
continue; |
|
| 509 |
} |
|
| 510 |
|
|
| 511 |
if (format != null && format.equals(MediaType.APPLICATION_JSON)) {
|
|
| 512 |
searchSolrResults.add(SolrResultsFormatter.xml2Json(result)); |
|
| 513 |
} else { // default xml*/
|
|
| 514 |
searchSolrResults.add(result); |
|
| 515 |
/*}*/ |
|
| 516 |
} |
|
| 517 |
|
|
| 518 |
estimatedTime = System.nanoTime() - startTime; |
|
| 519 |
logger.info("Internal transformation time " + estimatedTime/1000000 + " milliseconds for query:" + queryOpts.get("q") +
|
|
| 520 |
" and facets " + queryOpts.getAll("facet.field") + " and fq " + queryOpts.getAll("fq") + " from: "
|
|
| 521 |
+ from + " and size " + to); |
|
| 522 |
|
|
| 523 |
Map<String,List<String>> response = new HashMap<String, List<String>>(); |
|
| 524 |
|
|
| 525 |
//logger.debug("refine results " + refineSolrResults);
|
|
| 526 |
//logger.info("search results SIZE " + searchSolrResults.size());
|
|
| 527 |
//logger.info("search results " + searchSolrResults);
|
|
| 528 |
|
|
| 529 |
|
|
| 530 |
response.put("refine",refineSolrResults);
|
|
| 531 |
response.put("search", searchSolrResults);
|
|
| 532 |
|
|
| 533 |
return response; |
|
| 534 |
|
|
| 535 |
} catch (SolrServerException sse) {
|
|
| 536 |
logger.error("Error calling Solr for query:" + queryOpts.get("q") + " and facets "
|
|
| 537 |
+ queryOpts.getAll("facet.field") + " and fq " + queryOpts.getAll("fq") + " from: " + from +
|
|
| 538 |
" and size " + to, sse); |
|
| 539 |
throw new SearchServiceException("Error contacting Solr.", sse);
|
|
| 540 |
|
|
| 541 |
} catch (IOException ioe) {
|
|
| 542 |
logger.error("Error calling Solr for query:" + queryOpts.get("q") + " and facets "
|
|
| 543 |
+ queryOpts.getAll("facet.field") + " and fq " + queryOpts.getAll("fq") + " from: " + from +
|
|
| 544 |
" and size " + to, ioe); |
|
| 545 |
throw new SearchServiceException("Error reading results from Solr.", ioe);
|
|
| 546 |
|
|
| 547 |
} |
|
| 548 |
} |
|
| 549 |
|
|
| 337 | 550 |
/** |
| 338 | 551 |
* limit is the maximum number of results the cursor get is allowed to fetch. If limit is set to -1 all |
| 339 | 552 |
* results are returned. |
| 340 | 553 |
*/ |
| 341 |
public void cursorGet(Transformer transformer, int limit, OutputStream os) throws SolrServerException, SearchServiceException {
|
|
| 554 |
public void cursorGet(Transformer transformer, boolean special, boolean hasTitle, boolean isHtml, |
|
| 555 |
int limit, OutputStream os) throws SolrServerException, SearchServiceException {
|
|
| 342 | 556 |
|
| 557 |
System.out.println("Datasource Mapper " + datasourceCsvMapper);
|
|
| 558 |
|
|
| 343 | 559 |
int rows = 500; |
| 344 | 560 |
int limitCounter = -1; |
| 345 | 561 |
|
| ... | ... | |
| 347 | 563 |
queryOpts.add("rows", "0");
|
| 348 | 564 |
queryOpts.remove("rows");
|
| 349 | 565 |
queryOpts.add("rows", rows+"");
|
| 350 |
queryOpts.add("fl", "__result");
|
|
| 566 |
//queryOpts.add("fl", "__result");
|
|
| 351 | 567 |
queryOpts.add("shards.tolerant","true");
|
| 352 | 568 |
queryOpts.add("cursorMark", "*");
|
| 353 | 569 |
queryOpts.add("sort", "__indexrecordidentifier asc");
|
| ... | ... | |
| 369 | 585 |
cursorMark = nextCursorMark; |
| 370 | 586 |
nextCursorMark = resp.getNextCursorMark(); |
| 371 | 587 |
|
| 588 |
StringBuilder htmlBuilder = null; |
|
| 589 |
|
|
| 590 |
if(isHtml) {
|
|
| 591 |
htmlBuilder = new StringBuilder(); |
|
| 592 |
// Start the HTML table and headers |
|
| 593 |
htmlBuilder.append("<table>")
|
|
| 594 |
.append("<thead><tr>")
|
|
| 595 |
.append("<th>Type</th>" +
|
|
| 596 |
"<th>Title</th>" + |
|
| 597 |
"<th>Authors</th>" + |
|
| 598 |
"<th>Publication Year</th>" + |
|
| 599 |
"<th>DOI</th>" + |
|
| 600 |
"<th>Permanent Identifier</th>" + |
|
| 601 |
"<th>Publication type</th>" + |
|
| 602 |
"<th>Journal</th>" + |
|
| 603 |
"<th>Project Name (GA Number)</th>" + |
|
| 604 |
"<th>Access Mode</th>") // Add more headers as needed |
|
| 605 |
.append("</tr></thead><tbody>");
|
|
| 606 |
os.write(htmlBuilder.toString().getBytes()); |
|
| 607 |
os.flush(); |
|
| 608 |
} |
|
| 372 | 609 |
for (int i = 0; i < resp.getResults().size(); i++) {
|
| 373 |
if (transformer != null) {
|
|
| 374 |
String result = null; |
|
| 375 |
try {
|
|
| 376 |
logger.debug("PRE RESULT " + resp.getResults().get(i).get("__result"));
|
|
| 377 |
result = transformer.transform((String) resp.getResults().get(i).get("__result"));
|
|
| 378 |
logger.debug("RESULT " + result);
|
|
| 610 |
SolrRecord record = objectMapper.readValue((String) resp.getResults().get(i).get("__json"), new TypeReference<SolrRecord>() {});
|
|
| 611 |
if(!isHtml) {
|
|
| 612 |
if (transformer != null) {
|
|
| 613 |
String result = null; |
|
| 379 | 614 |
|
| 380 |
} catch (TransformerException te) {
|
|
| 381 |
logger.warn("Error transforming " + result, te);
|
|
| 382 |
continue; |
|
| 383 |
} |
|
| 384 | 615 |
|
| 385 |
try {
|
|
| 386 |
os.write(result.getBytes()); |
|
| 387 |
os.flush(); |
|
| 388 |
} catch (IOException e) {
|
|
| 389 |
logger.error("Cursor get... ", e);
|
|
| 390 |
continue; |
|
| 616 |
Datasource datasource = record.getDatasource(); |
|
| 617 |
Project project = record.getProject(); |
|
| 618 |
Result result1 = record.getResult(); |
|
| 619 |
Organization organization = record.getOrganization(); |
|
| 620 |
|
|
| 621 |
System.out.println("datasource " + datasource);
|
|
| 622 |
System.out.println("project " + project);
|
|
| 623 |
System.out.println("result " + result);
|
|
| 624 |
System.out.println("organization " + organization);
|
|
| 625 |
|
|
| 626 |
CsvEntity csvEntity = null; |
|
| 627 |
String csvRow = ""; |
|
| 628 |
ObjectWriter csvWriterRow = null; |
|
| 629 |
|
|
| 630 |
if (datasource != null) {
|
|
| 631 |
csvEntity = datasourceCsvMapper.toDatasourceCsv(record); |
|
| 632 |
CsvSchema schema = null; |
|
| 633 |
if (hasTitle) |
|
| 634 |
schema = datasourceMapper.schemaFor(DatasourceCsv.class).withHeader(); |
|
| 635 |
else |
|
| 636 |
schema = datasourceMapper.schemaFor(DatasourceCsv.class).withoutHeader(); |
|
| 637 |
//ObjectWriter csvWriterHeader = datasourceMapper.writer(schema); // Initialize for header |
|
| 638 |
csvWriterRow = datasourceMapper.writer(schema.withoutHeader()); |
|
| 639 |
} else if (organization != null) {
|
|
| 640 |
csvEntity = organizationCsvMapper.toOrganizationCsv(record); |
|
| 641 |
CsvSchema schema = null; |
|
| 642 |
if (hasTitle) |
|
| 643 |
schema = organizationMapper.schemaFor(OrganizationCsv.class).withHeader(); |
|
| 644 |
else |
|
| 645 |
schema = organizationMapper.schemaFor(OrganizationCsv.class).withoutHeader(); |
|
| 646 |
//ObjectWriter csvWriterHeader = organizationMapper.writer(schema); // Initialize for header |
|
| 647 |
csvWriterRow = organizationMapper.writer(schema.withoutHeader()); |
|
| 648 |
} else if (project != null) {
|
|
| 649 |
csvEntity = projectCsvMapper.toProjectCsv(record); |
|
| 650 |
CsvSchema schema = null; |
|
| 651 |
if (hasTitle) |
|
| 652 |
schema = projectMapper.schemaFor(ProjectCsv.class).withHeader(); |
|
| 653 |
else |
|
| 654 |
schema = projectMapper.schemaFor(ProjectCsv.class).withoutHeader(); |
|
| 655 |
//ObjectWriter csvWriterHeader = projectMapper.writer(schema); // Initialize for header |
|
| 656 |
csvWriterRow = projectMapper.writer(schema.withoutHeader()); |
|
| 657 |
} else if (result1 != null) {
|
|
| 658 |
csvEntity = researchProductCsvMapper.toResearchProductCsv(record); |
|
| 659 |
CsvSchema schema = null; |
|
| 660 |
if (hasTitle) |
|
| 661 |
schema = researchProductMapper.schemaFor(ResearchProductCsv.class).withHeader(); |
|
| 662 |
else |
|
| 663 |
schema = researchProductMapper.schemaFor(ResearchProductCsv.class).withoutHeader(); |
|
| 664 |
//ObjectWriter csvWriterHeader = researchProductMapper.writer(schema); // Initialize for header |
|
| 665 |
csvWriterRow = researchProductMapper.writer(schema.withoutHeader()); |
|
| 666 |
} |
|
| 667 |
|
|
| 668 |
try {
|
|
| 669 |
if (!special) {
|
|
| 670 |
csvRow = csvWriterRow.writeValueAsString(csvEntity); |
|
| 671 |
os.write(csvRow.getBytes()); |
|
| 672 |
os.flush(); |
|
| 673 |
} else {
|
|
| 674 |
List<ResearchProductCsv> flattenedProducts = researchProductCsvConverter.flattenProjects(record); |
|
| 675 |
for (ResearchProductCsv internalCsvEntity : flattenedProducts) {
|
|
| 676 |
csvRow = csvWriterRow.writeValueAsString(internalCsvEntity); |
|
| 677 |
os.write(csvRow.getBytes()); |
|
| 678 |
os.flush(); |
|
| 679 |
} |
|
| 680 |
} |
|
| 681 |
} catch (IOException e) {
|
|
| 682 |
logger.error("Cursor get... ", e);
|
|
| 683 |
System.out.println("ERROR " + e.getMessage());
|
|
| 684 |
continue; |
|
| 685 |
} |
|
| 391 | 686 |
} |
| 687 |
} else {
|
|
| 688 |
ResearchProductHtml researchProductHtml = researchProductsHtmlMapper.toResearchProductHtml(record); |
|
| 689 |
htmlBuilder.append("<tr>")
|
|
| 690 |
.append("<td>").append(researchProductHtml.getType()).append("</td>")
|
|
| 691 |
.append("<td>").append(researchProductHtml.getTitle()).append("</td>")
|
|
| 692 |
.append("<td>").append(researchProductHtml.getAuthors()).append("</td>")
|
|
| 693 |
.append("<td>").append(researchProductHtml.getPublicationYear()).append("</td>")
|
|
| 694 |
.append("<td>").append(researchProductHtml.getDoi()).append("</td>")
|
|
| 695 |
.append("<td>").append(researchProductHtml.getPermanentIdentifier()).append("</td>")
|
|
| 696 |
.append("<td>").append(researchProductHtml.getPublicationType()).append("</td>")
|
|
| 697 |
.append("<td>").append(researchProductHtml.getJournal()).append("</td>")
|
|
| 698 |
.append("<td>").append(researchProductHtml.getProjectName_gaNumber()).append("</td>")
|
|
| 699 |
.append("<td>").append(researchProductHtml.getAccessMode()).append("</td>")
|
|
Also available in: Unified diff
json parser for portal apis + csv and html download