Revision 50143
Added by Claudio Atzori over 6 years ago
| modules/dnet-openaire-exporter/branches/dsm/src/main/java/eu/dnetlib/openaire/exporter/model/datasource/Response.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.openaire.exporter.model.datasource; |
|
| 2 |
|
|
| 3 |
import com.fasterxml.jackson.annotation.JsonAutoDetect; |
|
| 4 |
import io.swagger.annotations.ApiModel; |
|
| 5 |
import io.swagger.annotations.ApiModelProperty; |
|
| 6 |
|
|
| 7 |
/** |
|
| 8 |
* Created by claudio on 01/12/2016. |
|
| 9 |
*/ |
|
| 10 |
@JsonAutoDetect |
|
| 11 |
@ApiModel(value = "Api response model", description = "Api response model, provides a response header") |
|
| 12 |
public abstract class Response {
|
|
| 13 |
|
|
| 14 |
@ApiModelProperty(position = 0) |
|
| 15 |
private Header responseHeader; |
|
| 16 |
|
|
| 17 |
public Response() {
|
|
| 18 |
responseHeader = new Header(); |
|
| 19 |
} |
|
| 20 |
|
|
| 21 |
public Header getResponseHeader() {
|
|
| 22 |
return responseHeader; |
|
| 23 |
} |
|
| 24 |
|
|
| 25 |
public Response setResponseHeader(final Header responseHeader) {
|
|
| 26 |
this.responseHeader = responseHeader; |
|
| 27 |
return this; |
|
| 28 |
} |
|
| 29 |
|
|
| 30 |
} |
|
| modules/dnet-openaire-exporter/branches/dsm/src/main/java/eu/dnetlib/openaire/exporter/model/datasource/Header.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.openaire.exporter.model.datasource; |
|
| 2 |
|
|
| 3 |
import com.fasterxml.jackson.annotation.JsonAutoDetect; |
|
| 4 |
import io.swagger.annotations.ApiModel; |
|
| 5 |
|
|
| 6 |
/** |
|
| 7 |
* Created by claudio on 24/11/2016. |
|
| 8 |
*/ |
|
| 9 |
@ApiModel |
|
| 10 |
@JsonAutoDetect |
|
| 11 |
public class Header {
|
|
| 12 |
|
|
| 13 |
private long queryTime; |
|
| 14 |
|
|
| 15 |
private String datasourceId; |
|
| 16 |
|
|
| 17 |
private int statusCode; |
|
| 18 |
|
|
| 19 |
private String error = ""; |
|
| 20 |
|
|
| 21 |
public Header() {
|
|
| 22 |
} |
|
| 23 |
|
|
| 24 |
public Header setQueryTime(final long queryTime) {
|
|
| 25 |
this.queryTime = queryTime; |
|
| 26 |
return this; |
|
| 27 |
} |
|
| 28 |
|
|
| 29 |
public Header setDatasourceId(final String datasourceId) {
|
|
| 30 |
this.datasourceId = datasourceId; |
|
| 31 |
return this; |
|
| 32 |
} |
|
| 33 |
|
|
| 34 |
public Header setError(final String error) {
|
|
| 35 |
this.error = error; |
|
| 36 |
return this; |
|
| 37 |
} |
|
| 38 |
|
|
| 39 |
public long getQueryTime() {
|
|
| 40 |
return queryTime; |
|
| 41 |
} |
|
| 42 |
|
|
| 43 |
public String getDatasourceId() {
|
|
| 44 |
return datasourceId; |
|
| 45 |
} |
|
| 46 |
|
|
| 47 |
public String getError() {
|
|
| 48 |
return error; |
|
| 49 |
} |
|
| 50 |
|
|
| 51 |
public int getStatusCode() {
|
|
| 52 |
return statusCode; |
|
| 53 |
} |
|
| 54 |
|
|
| 55 |
public Header setStatusCode(final int statusCode) {
|
|
| 56 |
this.statusCode = statusCode; |
|
| 57 |
return this; |
|
| 58 |
} |
|
| 59 |
} |
|
| modules/dnet-openaire-exporter/branches/dsm/src/main/java/eu/dnetlib/openaire/exporter/datasource/DatasourceManagerRMI.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.openaire.exporter.datasource; |
|
| 2 |
|
|
| 3 |
import eu.dnetlib.enabling.datasources.common.Api; |
|
| 4 |
import eu.dnetlib.enabling.datasources.common.Datasource; |
|
| 5 |
import eu.dnetlib.enabling.datasources.common.DatasourceManagerCommon; |
|
| 6 |
import eu.dnetlib.enabling.datasources.common.DatasourceManagerException; |
|
| 7 |
import eu.dnetlib.openaire.exporter.datasource.clients.Response; |
|
| 8 |
|
|
| 9 |
public interface DatasourceManagerRMI<DS extends Datasource<?, ?>, API extends Api<?>> extends DatasourceManagerCommon<DS, API> {
|
|
| 10 |
|
|
| 11 |
Response listCountries() throws DatasourceManagerException; |
|
| 12 |
|
|
| 13 |
Response listIds(int page, int size) throws DatasourceManagerException; |
|
| 14 |
|
|
| 15 |
Response getDsInfo(String dsId) throws DatasourceManagerException; |
|
| 16 |
|
|
| 17 |
Response searchByName(String name, int page, int size) throws DatasourceManagerException; |
|
| 18 |
|
|
| 19 |
Response searchByContactemail(String contactemail, int page, int size) throws DatasourceManagerException; |
|
| 20 |
|
|
| 21 |
Response searchByCountry(String country, int page, int size) throws DatasourceManagerException; |
|
| 22 |
|
|
| 23 |
void updateDatasourceName(String dsId, String officialname, String englishname) throws DatasourceManagerException; |
|
| 24 |
|
|
| 25 |
void updateCoordinates(String dsId, Double latitude, Double longitude) throws DatasourceManagerException; |
|
| 26 |
|
|
| 27 |
void updateBaseUrl(String apiId, String baseUrl) throws DatasourceManagerException; |
|
| 28 |
|
|
| 29 |
void updateTimezone(String dsId, String timezone) throws DatasourceManagerException; |
|
| 30 |
|
|
| 31 |
void updateTypology(String dsId, String timezone) throws DatasourceManagerException; |
|
| 32 |
|
|
| 33 |
void updateRegisteringUser(String dsId, String registeredBy) throws DatasourceManagerException; |
|
| 34 |
|
|
| 35 |
void updatePlatform(String dsId, String platform) throws DatasourceManagerException; |
|
| 36 |
|
|
| 37 |
|
|
| 38 |
} |
|
| modules/dnet-openaire-exporter/branches/dsm/src/main/java/eu/dnetlib/openaire/exporter/datasource/clients/Response.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.openaire.exporter.datasource.clients; |
|
| 2 |
|
|
| 3 |
import java.util.List; |
|
| 4 |
|
|
| 5 |
import com.fasterxml.jackson.annotation.JsonAutoDetect; |
|
| 6 |
import eu.dnetlib.openaire.exporter.model.datasource.DatasourceInfo; |
|
| 7 |
import eu.dnetlib.openaire.exporter.model.datasource.db.ApiDbEntry; |
|
| 8 |
import eu.dnetlib.openaire.exporter.vocabularies.Country; |
|
| 9 |
import io.swagger.annotations.ApiModel; |
|
| 10 |
import io.swagger.annotations.ApiModelProperty; |
|
| 11 |
|
|
| 12 |
/** |
|
| 13 |
* Created by claudio on 30/11/2016. |
|
| 14 |
*/ |
|
| 15 |
@JsonAutoDetect |
|
| 16 |
@ApiModel(value = "DatasourceManager response model", description = "provides information about the DatasourceManager responses") |
|
| 17 |
public class Response {
|
|
| 18 |
|
|
| 19 |
@ApiModelProperty(position = 1) |
|
| 20 |
private Header header; |
|
| 21 |
|
|
| 22 |
@ApiModelProperty(position = 2) |
|
| 23 |
private List<DatasourceInfo> datasourceInfo; |
|
| 24 |
|
|
| 25 |
@ApiModelProperty(position = 3) |
|
| 26 |
private List<ApiDbEntry> apiResponse; |
|
| 27 |
|
|
| 28 |
@ApiModelProperty(position = 4) |
|
| 29 |
private List<Country> countries; |
|
| 30 |
|
|
| 31 |
@ApiModelProperty(position = 5) |
|
| 32 |
private List<String> datasourceIds; |
|
| 33 |
|
|
| 34 |
public Header getHeader() {
|
|
| 35 |
return this.header; |
|
| 36 |
} |
|
| 37 |
|
|
| 38 |
public Response setHeader(final Header header) {
|
|
| 39 |
this.header = header; |
|
| 40 |
return this; |
|
| 41 |
} |
|
| 42 |
|
|
| 43 |
public List<DatasourceInfo> getDatasourceInfo() {
|
|
| 44 |
return datasourceInfo; |
|
| 45 |
} |
|
| 46 |
|
|
| 47 |
public Response setDatasourceInfo(final List<DatasourceInfo> datasourceInfo) {
|
|
| 48 |
this.datasourceInfo = datasourceInfo; |
|
| 49 |
return this; |
|
| 50 |
} |
|
| 51 |
|
|
| 52 |
public List<ApiDbEntry> getApiResponse() {
|
|
| 53 |
return this.apiResponse; |
|
| 54 |
} |
|
| 55 |
|
|
| 56 |
public Response setApiResponse(final List<ApiDbEntry> apiResponse) {
|
|
| 57 |
this.apiResponse = apiResponse; |
|
| 58 |
return this; |
|
| 59 |
} |
|
| 60 |
|
|
| 61 |
public List<Country> getCountries() {
|
|
| 62 |
return this.countries; |
|
| 63 |
} |
|
| 64 |
|
|
| 65 |
public Response setCountries(final List<Country> countries) {
|
|
| 66 |
this.countries = countries; |
|
| 67 |
return this; |
|
| 68 |
} |
|
| 69 |
|
|
| 70 |
public List<String> getDatasourceIds() {
|
|
| 71 |
return datasourceIds; |
|
| 72 |
} |
|
| 73 |
|
|
| 74 |
public Response setDatasourceIds(final List<String> datasourceIds) {
|
|
| 75 |
this.datasourceIds = datasourceIds; |
|
| 76 |
return this; |
|
| 77 |
} |
|
| 78 |
} |
|
| 79 |
|
|
| modules/dnet-openaire-exporter/branches/dsm/src/main/java/eu/dnetlib/openaire/exporter/datasource/clients/Header.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.openaire.exporter.datasource.clients; |
|
| 2 |
|
|
| 3 |
import java.util.List; |
|
| 4 |
|
|
| 5 |
import com.fasterxml.jackson.annotation.JsonAutoDetect; |
|
| 6 |
import io.swagger.annotations.ApiModel; |
|
| 7 |
import io.swagger.annotations.ApiModelProperty; |
|
| 8 |
|
|
| 9 |
@JsonAutoDetect |
|
| 10 |
@ApiModel(value = "DatasourceManager response header", description = "provides information about the DatasourceManager responses") |
|
| 11 |
public class Header {
|
|
| 12 |
|
|
| 13 |
@ApiModelProperty(position = 1) |
|
| 14 |
private Long time; |
|
| 15 |
|
|
| 16 |
@ApiModelProperty(position = 2) |
|
| 17 |
private int status; |
|
| 18 |
|
|
| 19 |
@ApiModelProperty(position = 3) |
|
| 20 |
private List<String> errors; |
|
| 21 |
|
|
| 22 |
public Long getTime() {
|
|
| 23 |
return this.time; |
|
| 24 |
} |
|
| 25 |
|
|
| 26 |
public int getStatus() {
|
|
| 27 |
return this.status; |
|
| 28 |
} |
|
| 29 |
|
|
| 30 |
public List<String> getErrors() {
|
|
| 31 |
return this.errors; |
|
| 32 |
} |
|
| 33 |
|
|
| 34 |
public Header setTime(final Long time) {
|
|
| 35 |
this.time = time; |
|
| 36 |
return this; |
|
| 37 |
} |
|
| 38 |
|
|
| 39 |
public Header setStatus(final int status) {
|
|
| 40 |
this.status = status; |
|
| 41 |
return this; |
|
| 42 |
} |
|
| 43 |
|
|
| 44 |
public Header setErrors(final List<String> errors) {
|
|
| 45 |
this.errors = errors; |
|
| 46 |
return this; |
|
| 47 |
} |
|
| 48 |
|
|
| 49 |
} |
|
| modules/dnet-openaire-exporter/branches/dsm/src/main/java/eu/dnetlib/openaire/exporter/funders/FundersApiController.java | ||
|---|---|---|
| 26 | 26 |
@Autowired |
| 27 | 27 |
private FunderDao fDao; |
| 28 | 28 |
|
| 29 |
@RequestMapping(value = "/funders", produces = { "application/json" }, method = RequestMethod.GET)
|
|
| 30 |
@ApiOperation( |
|
| 31 |
value = "get basic information about funders", |
|
| 32 |
notes = "basic information about funders: id, name, shortname, last update date, registration date", |
|
| 33 |
response = FunderDetails[].class) |
|
| 34 |
@ApiResponses(value = {
|
|
| 35 |
@ApiResponse(code = 200, message = "OK", response = FunderDetails[].class), |
|
| 36 |
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) |
|
| 37 |
public List<FunderDetails> getFunders() throws FundersApiException {
|
|
| 38 |
|
|
| 39 |
return fDao.listFunderDetails(); |
|
| 40 |
} |
|
| 41 |
|
|
| 29 | 42 |
@RequestMapping(value = "/funder/{id}", produces = { "application/json" }, method = RequestMethod.GET)
|
| 30 |
@ApiOperation(value = "get the funder details", notes = "get the funder details", response = FunderDetails.class)
|
|
| 43 |
@ApiOperation(value = "get the funder details", notes = "complete funder information", response = FunderDetails.class)
|
|
| 31 | 44 |
@ApiResponses(value = {
|
| 32 | 45 |
@ApiResponse(code = 200, message = "OK", response = FunderDetails.class), |
| 33 | 46 |
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) |
| 34 | 47 |
public FunderDetails getFunderDetails( |
| 35 | 48 |
@PathVariable final String id) throws FundersApiException {
|
| 36 | 49 |
|
| 37 |
return fDao.getFunderDetails(id); |
|
| 50 |
return fDao.getExtendedFunderDetails(id);
|
|
| 38 | 51 |
} |
| 39 | 52 |
|
| 40 | 53 |
@RequestMapping(value = "/funder/ids", produces = { "application/json" }, method = RequestMethod.GET)
|
| ... | ... | |
| 47 | 60 |
return fDao.listFunderIds(); |
| 48 | 61 |
} |
| 49 | 62 |
|
| 63 |
@RequestMapping(value = "/funder/dropcache", method = RequestMethod.GET) |
|
| 64 |
public void dropCache() throws FundersApiException {
|
|
| 65 |
|
|
| 66 |
fDao.dropCache(); |
|
| 67 |
} |
|
| 68 |
|
|
| 50 | 69 |
} |
| modules/dnet-openaire-exporter/branches/dsm/dnet-openaire-exporter.iml | ||
|---|---|---|
| 19 | 19 |
<orderEntry type="sourceFolder" forTests="false" /> |
| 20 | 20 |
<orderEntry type="library" name="Maven: eu.dnetlib:cnr-rmi-api:2.6.2-SNAPSHOT" level="project" /> |
| 21 | 21 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-core:3.1.5" level="project" /> |
| 22 |
<orderEntry type="library" name="Maven: org.codehaus.woodstox:woodstox-core-asl:4.4.1" level="project" /> |
|
| 23 |
<orderEntry type="library" name="Maven: org.codehaus.woodstox:stax2-api:3.1.4" level="project" /> |
|
| 24 | 22 |
<orderEntry type="library" name="Maven: org.apache.ws.xmlschema:xmlschema-core:2.2.1" level="project" /> |
| 25 | 23 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-frontend-jaxws:3.1.5" level="project" /> |
| 26 | 24 |
<orderEntry type="library" name="Maven: xml-resolver:xml-resolver:1.2" level="project" /> |
| ... | ... | |
| 65 | 63 |
<orderEntry type="library" name="Maven: eu.dnetlib:dnet-pace-core:2.5.3-SNAPSHOT" level="project" /> |
| 66 | 64 |
<orderEntry type="library" name="Maven: edu.cmu:secondstring:1.0.0" level="project" /> |
| 67 | 65 |
<orderEntry type="library" name="Maven: commons-collections:commons-collections:3.2.2" level="project" /> |
| 68 |
<orderEntry type="library" name="Maven: eu.dnetlib:dnet-index-solr-common:1.3.1" level="project" />
|
|
| 66 |
<orderEntry type="library" name="Maven: eu.dnetlib:dnet-index-solr-common:1.3.2-SNAPSHOT" level="project" />
|
|
| 69 | 67 |
<orderEntry type="library" name="Maven: com.googlecode.protobuf-java-format:protobuf-java-format:1.2" level="project" /> |
| 70 | 68 |
<orderEntry type="library" name="Maven: eu.dnetlib:dnet-objectstore-rmi:2.0.11-SNAPSHOT" level="project" /> |
| 71 |
<orderEntry type="library" name="Maven: org.apache.solr:solr-solrj:4.10.4" level="project" /> |
|
| 72 |
<orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.4.6" level="project" /> |
|
| 73 |
<orderEntry type="library" name="Maven: org.codehaus.woodstox:wstx-asl:3.2.7" level="project" /> |
|
| 74 |
<orderEntry type="library" name="Maven: org.noggit:noggit:0.5" level="project" /> |
|
| 69 |
<orderEntry type="library" name="Maven: org.apache.solr:solr-solrj:7.1.0" level="project" /> |
|
| 70 |
<orderEntry type="library" name="Maven: org.apache.commons:commons-math3:3.6.1" level="project" /> |
|
| 71 |
<orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.4.10" level="project" /> |
|
| 72 |
<orderEntry type="library" name="Maven: org.codehaus.woodstox:stax2-api:3.1.4" level="project" /> |
|
| 73 |
<orderEntry type="library" name="Maven: org.codehaus.woodstox:woodstox-core-asl:4.4.1" level="project" /> |
|
| 74 |
<orderEntry type="library" name="Maven: org.noggit:noggit:0.8" level="project" /> |
|
| 75 |
<orderEntry type="library" name="Maven: org.slf4j:jcl-over-slf4j:1.7.24" level="project" /> |
|
| 75 | 76 |
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.24" level="project" /> |
| 76 | 77 |
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-web:1.5.2.RELEASE" level="project" /> |
| 77 | 78 |
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter:1.5.2.RELEASE" level="project" /> |
| ... | ... | |
| 111 | 112 |
<orderEntry type="library" name="Maven: org.springframework.data:spring-data-jpa:1.11.1.RELEASE" level="project" /> |
| 112 | 113 |
<orderEntry type="library" name="Maven: org.springframework.data:spring-data-commons:1.13.1.RELEASE" level="project" /> |
| 113 | 114 |
<orderEntry type="library" name="Maven: org.springframework:spring-orm:4.3.7.RELEASE" level="project" /> |
| 114 |
<orderEntry type="library" name="Maven: org.slf4j:jcl-over-slf4j:1.7.24" level="project" /> |
|
| 115 | 115 |
<orderEntry type="library" name="Maven: org.springframework:spring-aspects:4.3.7.RELEASE" level="project" /> |
| 116 | 116 |
<orderEntry type="library" name="Maven: org.postgresql:postgresql:42.0.0" level="project" /> |
| 117 | 117 |
<orderEntry type="library" name="Maven: org.apache.commons:commons-dbcp2:2.1" level="project" /> |
| modules/dnet-openaire-exporter/branches/dsm/src/main/resources/eu/dnetlib/openaire/exporter/xquery/updateRepoField.xquery | ||
|---|---|---|
| 1 |
for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')
|
|
| 2 |
where $x/RESOURCE_PROFILE/BODY/CONFIGURATION/DATASOURCE_ORIGINAL_ID[@provenance="OPENAIRE"]/text() = "%s" |
|
| 3 |
return update value $x//%s with "%s" |
|
| modules/dnet-openaire-exporter/branches/dsm/pom.xml | ||
|---|---|---|
| 268 | 268 |
|
| 269 | 269 |
<properties> |
| 270 | 270 |
<java.version>1.8</java.version> |
| 271 |
<apache.solr.version>4.10.4</apache.solr.version>
|
|
| 271 |
<apache.solr.version>7.1.0</apache.solr.version>
|
|
| 272 | 272 |
<mongodb.driver.version>3.4.2</mongodb.driver.version> |
| 273 | 273 |
<springfox-version>2.7.0</springfox-version> |
| 274 | 274 |
<prometheus.version>0.0.25</prometheus.version> |
| modules/dnet-openaire-exporter/branches/dsm/src/main/java/eu/dnetlib/openaire/exporter/funders/FunderDao.java | ||
|---|---|---|
| 4 | 4 |
import java.util.Date; |
| 5 | 5 |
import java.util.List; |
| 6 | 6 |
import java.util.Map; |
| 7 |
import java.util.Queue; |
|
| 8 |
import java.util.concurrent.LinkedBlockingQueue; |
|
| 9 |
import java.util.function.Function; |
|
| 10 | 7 |
import java.util.stream.Collectors; |
| 11 | 8 |
|
| 12 | 9 |
import com.google.common.collect.Lists; |
| ... | ... | |
| 14 | 11 |
import eu.dnetlib.openaire.exporter.datasource.clients.MongoLoggerClient; |
| 15 | 12 |
import eu.dnetlib.openaire.exporter.funders.context.Context; |
| 16 | 13 |
import eu.dnetlib.openaire.exporter.funders.context.MappingUtils; |
| 14 |
import eu.dnetlib.openaire.exporter.funders.model.ExtendedFunderDetails; |
|
| 17 | 15 |
import eu.dnetlib.openaire.exporter.funders.model.FunderDetails; |
| 18 | 16 |
import org.apache.commons.collections.MapUtils; |
| 19 |
import org.apache.commons.lang3.StringUtils; |
|
| 20 | 17 |
import org.apache.commons.logging.Log; |
| 21 | 18 |
import org.apache.commons.logging.LogFactory; |
| 22 | 19 |
import org.springframework.beans.factory.annotation.Autowired; |
| 20 |
import org.springframework.cache.annotation.CacheEvict; |
|
| 23 | 21 |
import org.springframework.cache.annotation.Cacheable; |
| 24 | 22 |
import org.springframework.stereotype.Component; |
| 25 | 23 |
|
| ... | ... | |
| 34 | 32 |
@Autowired |
| 35 | 33 |
private MongoLoggerClient mongoLoggerClient; |
| 36 | 34 |
|
| 37 |
//TODO replace Exception with something dedicated to the Funders API |
|
| 38 |
@Cacheable(value = "funders-cache") |
|
| 39 |
public FunderDetails getFunderDetails(final String funderId) throws FundersApiException {
|
|
| 40 |
|
|
| 35 |
@Cacheable("funders-cache")
|
|
| 36 |
public ExtendedFunderDetails getExtendedFunderDetails(final String funderId) throws FundersApiException {
|
|
| 41 | 37 |
log.info(String.format("cache miss for getFunderDetails('%s')", funderId));
|
| 38 |
final Context c = getContext(funderId); |
|
| 39 |
final ExtendedFunderDetails details = MappingUtils.asExtendedFunderDetails(c); |
|
| 40 |
details.setLastUpdateDate(getLastUpdateDate(c.getId())); |
|
| 41 |
details.setRegistrationDate(getRegistrationDate(c.getId())); |
|
| 42 | 42 |
|
| 43 |
final Map<String, Context> contextMap = getFunderDetails(); |
|
| 44 |
if (MapUtils.isEmpty(contextMap)) {
|
|
| 45 |
throw new FundersApiException("Unable to find any context profile");
|
|
| 46 |
} |
|
| 43 |
return details; |
|
| 44 |
} |
|
| 47 | 45 |
|
| 48 |
final Context context = contextMap.get(funderId); |
|
| 49 |
if (context == null) {
|
|
| 50 |
throw new FundersApiException("Unexising funder id " + funderId);
|
|
| 46 |
public List<FunderDetails> listFunderDetails() throws FundersApiException {
|
|
| 47 |
try {
|
|
| 48 |
return isLookupClient.getContextMap().values().stream() |
|
| 49 |
.map(c -> asFunderDetails(c)) |
|
| 50 |
.collect(Collectors.toList()); |
|
| 51 |
} catch (IOException e) {
|
|
| 52 |
throw new FundersApiException(e); |
|
| 51 | 53 |
} |
| 52 |
return asFunderDetails(context); |
|
| 53 | 54 |
} |
| 54 | 55 |
|
| 55 | 56 |
public List<String> listFunderIds() throws FundersApiException {
|
| 56 |
return Lists.newArrayList(getFunderDetails().keySet()); |
|
| 57 |
} |
|
| 58 |
|
|
| 59 |
//TODO replace Exception with something dedicated to the Funders API |
|
| 60 |
@Cacheable(value = "funders-cache", key = "#root.method", unless = "#result == null or #result.isEmpty()") |
|
| 61 |
public Map<String, Context> getFunderDetails(final Queue<Throwable> errors) throws FundersApiException {
|
|
| 62 | 57 |
try {
|
| 63 |
return isLookupClient.getContextProfiles(errors).stream() |
|
| 64 |
.filter(StringUtils::isNotBlank) |
|
| 65 |
.map(s -> MappingUtils.parseContext(s, errors)) |
|
| 66 |
.collect(Collectors.toMap( |
|
| 67 |
Context::getId, |
|
| 68 |
Function.identity())); |
|
| 58 |
return Lists.newArrayList(isLookupClient.getContextMap().keySet()); |
|
| 69 | 59 |
} catch (IOException e) {
|
| 70 | 60 |
throw new FundersApiException(e); |
| 71 | 61 |
} |
| 72 | 62 |
} |
| 73 | 63 |
|
| 74 |
private Map<String, Context> getFunderDetails() throws FundersApiException {
|
|
| 75 |
return getFunderDetails(new LinkedBlockingQueue<>(100)); |
|
| 64 |
@CacheEvict(cacheNames = "funders-cache", allEntries = true) |
|
| 65 |
public void dropCache() {
|
|
| 66 |
log.info("dropped funders info cache");
|
|
| 76 | 67 |
} |
| 77 | 68 |
|
| 69 |
/* HELPERS */ |
|
| 70 |
|
|
| 78 | 71 |
private FunderDetails asFunderDetails(final Context c) {
|
| 79 | 72 |
return MappingUtils.asFunderDetails(c) |
| 80 | 73 |
.setLastUpdateDate(getLastUpdateDate(c.getId())) |
| ... | ... | |
| 89 | 82 |
return null; |
| 90 | 83 |
} |
| 91 | 84 |
|
| 85 |
private Context getContext(final String funderId) throws FundersApiException {
|
|
| 86 |
try {
|
|
| 87 |
final Map<String, Context> contextMap = isLookupClient.getContextMap(); |
|
| 88 |
if (MapUtils.isEmpty(contextMap)) {
|
|
| 89 |
throw new FundersApiException("Unable to find any context profile");
|
|
| 90 |
} |
|
| 92 | 91 |
|
| 92 |
final Context context = contextMap.get(funderId); |
|
| 93 |
if (context == null) {
|
|
| 94 |
throw new FundersApiException("Unexising funder id " + funderId);
|
|
| 95 |
} |
|
| 96 |
return context; |
|
| 97 |
} catch (IOException e) {
|
|
| 98 |
throw new FundersApiException(e); |
|
| 99 |
} |
|
| 100 |
} |
|
| 101 |
|
|
| 93 | 102 |
} |
| modules/dnet-openaire-exporter/branches/dsm/src/main/java/eu/dnetlib/openaire/exporter/datasource/repository/DatasourceDbEntryRepository.java | ||
|---|---|---|
| 3 | 3 |
import javax.transaction.Transactional; |
| 4 | 4 |
|
| 5 | 5 |
import eu.dnetlib.openaire.exporter.model.datasource.db.DatasourceDbEntry; |
| 6 |
import org.springframework.data.domain.PageRequest; |
|
| 7 |
import org.springframework.data.domain.Pageable; |
|
| 8 |
import org.springframework.data.domain.Slice; |
|
| 9 | 6 |
import org.springframework.data.jpa.repository.JpaRepository; |
| 7 |
import org.springframework.data.jpa.repository.JpaSpecificationExecutor; |
|
| 10 | 8 |
import org.springframework.data.jpa.repository.Modifying; |
| 11 | 9 |
import org.springframework.data.jpa.repository.Query; |
| 12 |
import org.springframework.scheduling.annotation.Async; |
|
| 13 | 10 |
import org.springframework.stereotype.Repository; |
| 14 |
import org.springframework.util.concurrent.ListenableFuture; |
|
| 15 | 11 |
|
| 16 | 12 |
/** |
| 17 | 13 |
* Created by claudio on 12/04/2017. |
| 18 | 14 |
*/ |
| 19 | 15 |
@Repository |
| 20 |
public interface DatasourceDbEntryRepository extends JpaRepository<DatasourceDbEntry, String> {
|
|
| 16 |
public interface DatasourceDbEntryRepository extends JpaRepository<DatasourceDbEntry, String>, JpaSpecificationExecutor<DatasourceDbEntry> {
|
|
| 21 | 17 |
|
| 22 | 18 |
DatasourceDbEntry findOneById(String id); |
| 23 | 19 |
|
| 24 |
@Async |
|
| 25 |
ListenableFuture<Slice<DatasourceDbEntry>> findDistinctByOfficialnameContainingOrEnglishnameContainingAllIgnoreCase( |
|
| 26 |
String officialname, |
|
| 27 |
String englishname, |
|
| 28 |
Pageable pageable); |
|
| 29 |
|
|
| 30 |
@Async |
|
| 31 |
ListenableFuture<Slice<DatasourceDbEntry>> findDistinctByContactemailContainingAllIgnoreCase(String contactemail, Pageable pageable); |
|
| 32 |
|
|
| 33 |
@Async |
|
| 34 |
ListenableFuture<Slice<DatasourceDbEntry>> findDistinctByOrganizationsCountryIgnoreCase(String country, Pageable pageable); |
|
| 35 |
|
|
| 36 |
@Async |
|
| 37 |
ListenableFuture<Slice<DatasourceDbEntry>> findDistinctByRegisteredbyIgnoreCase(String registeringUser, PageRequest pageRequest); |
|
| 38 |
|
|
| 39 | 20 |
@Query("select d.managed from #{#entityName} d where d.id = ?1")
|
| 40 | 21 |
boolean isManaged(String id); |
| 41 | 22 |
|
| modules/dnet-openaire-exporter/branches/dsm/src/main/java/eu/dnetlib/openaire/exporter/datasource/repository/DatasourceApiDbEntryRepository.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.openaire.exporter.datasource.repository; |
|
| 2 |
|
|
| 3 |
import eu.dnetlib.openaire.exporter.model.datasource.db.DatasourceApiDbEntry; |
|
| 4 |
import org.springframework.data.jpa.repository.JpaRepository; |
|
| 5 |
import org.springframework.data.jpa.repository.JpaSpecificationExecutor; |
|
| 6 |
import org.springframework.stereotype.Repository; |
|
| 7 |
|
|
| 8 |
@Repository |
|
| 9 |
public interface DatasourceApiDbEntryRepository extends JpaRepository<DatasourceApiDbEntry, String>, JpaSpecificationExecutor<DatasourceApiDbEntry> {
|
|
| 10 |
|
|
| 11 |
} |
|
| modules/dnet-openaire-exporter/branches/dsm/src/main/java/eu/dnetlib/openaire/exporter/datasource/clients/DatasourceIndexClient.java | ||
|---|---|---|
| 5 | 5 |
import java.util.List; |
| 6 | 6 |
import java.util.Map; |
| 7 | 7 |
import java.util.Queue; |
| 8 |
import java.util.concurrent.ConcurrentHashMap; |
|
| 9 |
import java.util.concurrent.CountDownLatch; |
|
| 10 |
import java.util.concurrent.ScheduledThreadPoolExecutor; |
|
| 11 |
import java.util.concurrent.TimeUnit; |
|
| 12 |
import java.util.concurrent.TimeoutException; |
|
| 13 |
|
|
| 8 |
import java.util.concurrent.*; |
|
| 14 | 9 |
import javax.annotation.PostConstruct; |
| 15 | 10 |
import javax.annotation.PreDestroy; |
| 16 | 11 |
|
| 12 |
import com.google.common.collect.Iterables; |
|
| 13 |
import com.google.common.util.concurrent.*; |
|
| 14 |
import eu.dnetlib.OpenaireExporterConfig; |
|
| 15 |
import eu.dnetlib.enabling.datasources.common.DatasourceManagerException; |
|
| 16 |
import eu.dnetlib.miscutils.functional.hash.Hashing; |
|
| 17 |
import eu.dnetlib.openaire.exporter.datasource.clients.utils.DatasourceFunctions; |
|
| 18 |
import eu.dnetlib.openaire.exporter.datasource.clients.utils.IndexDsInfo; |
|
| 19 |
import eu.dnetlib.openaire.exporter.datasource.clients.utils.IndexRecordsInfo; |
|
| 17 | 20 |
import org.apache.commons.lang.StringUtils; |
| 18 | 21 |
import org.apache.commons.lang.time.DateFormatUtils; |
| 19 |
import org.apache.commons.lang3.exception.ExceptionUtils; |
|
| 20 | 22 |
import org.apache.commons.logging.Log; |
| 21 | 23 |
import org.apache.commons.logging.LogFactory; |
| 24 |
import org.apache.solr.client.solrj.SolrQuery; |
|
| 25 |
import org.apache.solr.client.solrj.SolrServerException; |
|
| 26 |
import org.apache.solr.client.solrj.impl.CloudSolrClient; |
|
| 27 |
import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder; |
|
| 22 | 28 |
import org.apache.solr.client.solrj.response.QueryResponse; |
| 23 | 29 |
import org.apache.solr.common.SolrDocument; |
| 24 | 30 |
import org.springframework.beans.factory.annotation.Autowired; |
| 25 | 31 |
import org.springframework.http.HttpStatus; |
| 26 | 32 |
import org.springframework.stereotype.Component; |
| 27 | 33 |
|
| 28 |
import com.google.common.collect.Iterables; |
|
| 29 |
import com.google.common.util.concurrent.FutureCallback; |
|
| 30 |
import com.google.common.util.concurrent.Futures; |
|
| 31 |
import com.google.common.util.concurrent.ListeningExecutorService; |
|
| 32 |
import com.google.common.util.concurrent.MoreExecutors; |
|
| 33 |
import com.google.common.util.concurrent.ThreadFactoryBuilder; |
|
| 34 |
|
|
| 35 |
import eu.dnetlib.OpenaireExporterConfig; |
|
| 36 |
import eu.dnetlib.data.index.CloudIndexClient; |
|
| 37 |
import eu.dnetlib.data.index.CloudIndexClientException; |
|
| 38 |
import eu.dnetlib.data.index.CloudIndexClientFactory; |
|
| 39 |
import eu.dnetlib.enabling.datasources.common.DatasourceManagerException; |
|
| 40 |
import eu.dnetlib.miscutils.functional.hash.Hashing; |
|
| 41 |
import eu.dnetlib.openaire.exporter.datasource.clients.utils.DatasourceFunctions; |
|
| 42 |
import eu.dnetlib.openaire.exporter.datasource.clients.utils.IndexDsInfo; |
|
| 43 |
import eu.dnetlib.openaire.exporter.datasource.clients.utils.IndexRecordsInfo; |
|
| 44 |
|
|
| 45 | 34 |
/** |
| 46 | 35 |
* Created by claudio on 20/10/2016. |
| 47 | 36 |
*/ |
| ... | ... | |
| 58 | 47 |
|
| 59 | 48 |
private ListeningExecutorService executor; |
| 60 | 49 |
|
| 61 |
private static Map<String, CloudIndexClient> indexClientMap = new ConcurrentHashMap<>();
|
|
| 50 |
private static Map<String, CloudSolrClient> indexClientMap = new ConcurrentHashMap<>();
|
|
| 62 | 51 |
|
| 63 | 52 |
@PostConstruct |
| 64 | 53 |
public void init() {
|
| ... | ... | |
| 81 | 70 |
public IndexRecordsInfo getIndexInfo(final String dsId, final IndexDsInfo info, final Queue<Throwable> errors) throws DatasourceManagerException {
|
| 82 | 71 |
try {
|
| 83 | 72 |
final String collectedFrom = StringUtils.substringBefore(dsId, SEPARATOR) + SEPARATOR + Hashing.md5(StringUtils.substringAfter(dsId, SEPARATOR)); |
| 84 |
final CloudIndexClient indexClient = getIndexClient(info);
|
|
| 73 |
final CloudSolrClient indexClient = getIndexClient(info);
|
|
| 85 | 74 |
final CountDownLatch latch = new CountDownLatch(2); |
| 86 | 75 |
final IndexRecordsInfo indexRecordInfo = new IndexRecordsInfo(); |
| 87 | 76 |
|
| ... | ... | |
| 99 | 88 |
|
| 100 | 89 |
@Override |
| 101 | 90 |
public void onFailure(final Throwable e) {
|
| 102 |
log.error(ExceptionUtils.getStackTrace(e)); |
|
| 103 | 91 |
errors.offer(e); |
| 104 | 92 |
latch.countDown(); |
| 105 | 93 |
} |
| ... | ... | |
| 117 | 105 |
|
| 118 | 106 |
@Override |
| 119 | 107 |
public void onFailure(final Throwable e) {
|
| 120 |
log.error(ExceptionUtils.getStackTrace(e)); |
|
| 121 | 108 |
errors.offer(e); |
| 122 | 109 |
latch.countDown(); |
| 123 | 110 |
} |
| ... | ... | |
| 132 | 119 |
|
| 133 | 120 |
private Long setFunded( |
| 134 | 121 |
final String collectedFrom, |
| 135 |
final CloudIndexClient indexClient) throws DatasourceManagerException {
|
|
| 122 |
final CloudSolrClient indexClient) throws DatasourceManagerException {
|
|
| 136 | 123 |
final String query = |
| 137 | 124 |
String.format("oaftype:result AND deletedbyinference:false AND collectedfromdatasourceid:\"%s\" AND relprojectid:*", collectedFrom);
|
| 138 | 125 |
log.debug(String.format("query: %s", query));
|
| 139 | 126 |
try {
|
| 140 |
return indexClient.query(query, 0).getResults().getNumFound();
|
|
| 127 |
return indexClient.query(new SolrQuery(query).setRows(0)).getResults().getNumFound();
|
|
| 141 | 128 |
} catch (Throwable e) {
|
| 142 | 129 |
throw new DatasourceManagerException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying information system", e); |
| 143 | 130 |
} |
| ... | ... | |
| 145 | 132 |
|
| 146 | 133 |
private IndexRecordsInfo setDateAndTotal( |
| 147 | 134 |
final String collectedFrom, |
| 148 |
final CloudIndexClient indexClient) throws DatasourceManagerException {
|
|
| 135 |
final CloudSolrClient indexClient) throws DatasourceManagerException {
|
|
| 149 | 136 |
try {
|
| 150 | 137 |
final String query = String.format("oaftype:result AND deletedbyinference:false AND collectedfromdatasourceid:\"%s\"", collectedFrom);
|
| 151 | 138 |
log.debug(String.format("query: %s", query));
|
| 152 | 139 |
|
| 153 |
final QueryResponse rsp = indexClient.query(query, 1);
|
|
| 140 |
final QueryResponse rsp = indexClient.query(new SolrQuery(query).setRows(1));
|
|
| 154 | 141 |
final SolrDocument doc = Iterables.getFirst(rsp.getResults(), new SolrDocument()); |
| 155 | 142 |
if (log.isDebugEnabled()) {
|
| 156 |
log.debug(String.format("got document %s", doc.toString()));
|
|
| 143 |
log.debug(String.format("got document %s", doc.get("__indexrecordidentifier")));
|
|
| 157 | 144 |
} |
| 158 | 145 |
// if (doc.isEmpty()) {
|
| 159 | 146 |
// throw new DatasourceManagerException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("cannot find document matching
|
| ... | ... | |
| 178 | 165 |
|
| 179 | 166 |
public String getLastIndexingDate(final IndexDsInfo info) throws DatasourceManagerException {
|
| 180 | 167 |
try {
|
| 181 |
final QueryResponse rsp = getIndexClient(info).query("oaftype:datasource", 1);
|
|
| 168 |
final SolrQuery query = new SolrQuery("oaftype:datasource").setRows(1);
|
|
| 169 |
final QueryResponse rsp = getIndexClient(info).query(query); |
|
| 182 | 170 |
final SolrDocument doc = Iterables.getFirst(rsp.getResults(), null); |
| 183 | 171 |
final String dsversion = doc.get("__dsversion").toString();
|
| 184 | 172 |
return StringUtils.substringBefore(dsversion, "T"); |
| 185 |
} catch (CloudIndexClientException e) {
|
|
| 173 |
} catch (SolrServerException | IOException e) {
|
|
| 186 | 174 |
throw new DatasourceManagerException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying index DS profile: " + info, e); |
| 187 | 175 |
} |
| 188 | 176 |
} |
| 189 | 177 |
|
| 190 |
private synchronized CloudIndexClient getIndexClient(final IndexDsInfo info) throws CloudIndexClientException {
|
|
| 178 |
private synchronized CloudSolrClient getIndexClient(final IndexDsInfo info) {
|
|
| 191 | 179 |
if (!indexClientMap.containsKey(info.getColl())) {
|
| 192 |
indexClientMap.put(info.getColl(), CloudIndexClientFactory.newIndexClient(info.getIndexBaseUrl(), info.getColl(), false)); |
|
| 180 |
|
|
| 181 |
final CloudSolrClient client = new Builder().withZkHost(info.getIndexBaseUrl()).build(); |
|
| 182 |
client.setDefaultCollection(info.getColl()); |
|
| 183 |
|
|
| 184 |
indexClientMap.put(info.getColl(), client); |
|
| 193 | 185 |
} |
| 194 | 186 |
return indexClientMap.get(info.getColl()); |
| 195 | 187 |
} |
| modules/dnet-openaire-exporter/branches/dsm/src/main/java/eu/dnetlib/openaire/exporter/datasource/clients/DatasourceDao.java | ||
|---|---|---|
| 6 | 6 |
import eu.dnetlib.enabling.datasources.common.Datasource; |
| 7 | 7 |
import eu.dnetlib.enabling.datasources.common.DatasourceManagerCommon; |
| 8 | 8 |
import eu.dnetlib.enabling.datasources.common.DatasourceManagerException; |
| 9 |
import eu.dnetlib.openaire.exporter.model.datasource.RequestFilter; |
|
| 10 |
import eu.dnetlib.openaire.exporter.model.datasource.RequestSort; |
|
| 11 |
import eu.dnetlib.openaire.exporter.model.datasource.RequestSortOrder; |
|
| 9 | 12 |
import eu.dnetlib.openaire.exporter.vocabularies.Country; |
| 10 |
import org.springframework.data.domain.Slice; |
|
| 11 |
import org.springframework.util.concurrent.ListenableFuture; |
|
| 13 |
import org.springframework.data.domain.Page; |
|
| 12 | 14 |
|
| 13 | 15 |
public interface DatasourceDao<DS extends Datasource<?, ?>, API extends Api<?>> extends DatasourceManagerCommon<DS, API> {
|
| 14 | 16 |
|
| 17 |
// DATASOURCE |
|
| 18 |
|
|
| 15 | 19 |
List<Country> listCountries() throws DatasourceManagerException; |
| 16 | 20 |
|
| 17 | 21 |
List<String> listIds(int page, int size) throws DatasourceManagerException; |
| 18 | 22 |
|
| 19 |
ListenableFuture<Slice<DS>> searchByName(String name, int page, int size) throws DatasourceManagerException;
|
|
| 23 |
Page<DS> search(RequestSort requestSortBy, RequestSortOrder order, RequestFilter requestFilter, int page, int size) throws DatasourceManagerException;
|
|
| 20 | 24 |
|
| 21 |
ListenableFuture<Slice<DS>> searchByCountry(String country, int page, int size) throws DatasourceManagerException; |
|
| 22 |
|
|
| 23 |
ListenableFuture<Slice<DS>> searchByContactemail(String contactemail, int page, int size) throws DatasourceManagerException; |
|
| 24 |
|
|
| 25 |
ListenableFuture<Slice<DS>> searchByRegisteringUser(String registeringUser, int page, int size) throws DatasourceManagerException; |
|
| 26 |
|
|
| 27 | 25 |
void updateDatasourceName(String dsId, String officialname, String englishname) throws DatasourceManagerException; |
| 28 | 26 |
|
| 29 | 27 |
void updateCoordinates(String dsId, Double latitude, Double longitude) throws DatasourceManagerException; |
| 30 | 28 |
|
| 31 |
void updateBaseUrl(String apiId, String baseUrl) throws DatasourceManagerException; |
|
| 32 |
|
|
| 33 | 29 |
void updateTimezone(String dsId, String timezone) throws DatasourceManagerException; |
| 34 | 30 |
|
| 35 | 31 |
void updateTypology(String dsId, String timezone) throws DatasourceManagerException; |
| ... | ... | |
| 38 | 34 |
|
| 39 | 35 |
void updatePlatform(String dsId, String platform) throws DatasourceManagerException; |
| 40 | 36 |
|
| 37 |
// API |
|
| 38 |
|
|
| 39 |
List<String> findBaseURLs(RequestFilter requestFilter, int page, int size) throws DatasourceManagerException; |
|
| 40 |
|
|
| 41 |
void updateBaseUrl(String apiId, String baseUrl) throws DatasourceManagerException; |
|
| 41 | 42 |
} |
| modules/dnet-openaire-exporter/branches/dsm/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java | ||
|---|---|---|
| 4 | 4 |
import java.util.List; |
| 5 | 5 |
import java.util.Set; |
| 6 | 6 |
import java.util.stream.Collectors; |
| 7 |
|
|
| 8 | 7 |
import javax.servlet.http.HttpServletResponse; |
| 9 | 8 |
|
| 10 |
import org.apache.commons.logging.Log; |
|
| 11 |
import org.apache.commons.logging.LogFactory; |
|
| 12 | 9 |
import org.springframework.beans.factory.annotation.Autowired; |
| 13 | 10 |
import org.springframework.boot.SpringApplication; |
| 11 |
import org.springframework.boot.autoconfigure.EnableAutoConfiguration; |
|
| 14 | 12 |
import org.springframework.boot.autoconfigure.SpringBootApplication; |
| 13 |
import org.springframework.boot.autoconfigure.solr.SolrAutoConfiguration; |
|
| 15 | 14 |
import org.springframework.cache.annotation.EnableCaching; |
| 16 | 15 |
import org.springframework.context.annotation.EnableAspectJAutoProxy; |
| 17 | 16 |
import org.springframework.web.bind.annotation.RequestMapping; |
| ... | ... | |
| 23 | 22 |
@RestController |
| 24 | 23 |
@SpringBootApplication |
| 25 | 24 |
@EnableAspectJAutoProxy |
| 25 |
@EnableAutoConfiguration(exclude = { SolrAutoConfiguration.class })
|
|
| 26 | 26 |
public class DNetOpenaireExporterApplication {
|
| 27 | 27 |
|
| 28 |
private static final Log log = LogFactory.getLog(DNetOpenaireExporterApplication.class); |
|
| 29 |
|
|
| 30 | 28 |
@Autowired |
| 31 | 29 |
private List<RequestMappingHandlerMapping> handlerMappings; |
| 32 | 30 |
|
| modules/dnet-openaire-exporter/branches/dsm/src/main/resources/templates/dnetprofiles/repository.xml | ||
|---|---|---|
| 1 |
<RESOURCE_PROFILE> |
|
| 2 |
<HEADER> |
|
| 3 |
<RESOURCE_IDENTIFIER value=""/> |
|
| 4 |
<RESOURCE_TYPE value="RepositoryServiceResourceType"/> |
|
| 5 |
<RESOURCE_KIND value="RepositoryServiceResources"/> |
|
| 6 |
<RESOURCE_URI value=""/> |
|
| 7 |
<DATE_OF_CREATION value=""/> |
|
| 8 |
<PROTOCOL/> |
|
| 9 |
</HEADER> |
|
| 10 |
<BODY> |
|
| 11 |
<CONFIGURATION> |
|
| 12 |
<DATASOURCE_TYPE></DATASOURCE_TYPE> |
|
| 13 |
<DATASOURCE_ORIGINAL_ID provenance="OPENAIRE"></DATASOURCE_ORIGINAL_ID> |
|
| 14 |
<DATASOURCE_AGGREGATED>false</DATASOURCE_AGGREGATED> |
|
| 15 |
<ENVIRONMENTS> |
|
| 16 |
<ENVIRONMENT>OPENAIRE</ENVIRONMENT> |
|
| 17 |
</ENVIRONMENTS> |
|
| 18 |
<TYPOLOGY></TYPOLOGY> |
|
| 19 |
<MAX_SIZE_OF_DATASTRUCTURE>0</MAX_SIZE_OF_DATASTRUCTURE> |
|
| 20 |
<AVAILABLE_DISKSPACE>0</AVAILABLE_DISKSPACE> |
|
| 21 |
<MAX_NUMBER_OF_DATASTRUCTURE>0</MAX_NUMBER_OF_DATASTRUCTURE> |
|
| 22 |
<OFFICIAL_NAME></OFFICIAL_NAME> |
|
| 23 |
<ENGLISH_NAME></ENGLISH_NAME> |
|
| 24 |
<ICON_URI></ICON_URI> |
|
| 25 |
<COUNTRY></COUNTRY> |
|
| 26 |
<LOCATION> |
|
| 27 |
<LONGITUDE></LONGITUDE> |
|
| 28 |
<LATITUDE></LATITUDE> |
|
| 29 |
<TIMEZONE></TIMEZONE> |
|
| 30 |
</LOCATION> |
|
| 31 |
<REPOSITORY_WEBPAGE></REPOSITORY_WEBPAGE> |
|
| 32 |
<REPOSITORY_INSTITUTION></REPOSITORY_INSTITUTION> |
|
| 33 |
<ADMIN_INFO></ADMIN_INFO> |
|
| 34 |
<INTERFACES></INTERFACES> |
|
| 35 |
<EXTRA_FIELDS> |
|
| 36 |
<FIELD> |
|
| 37 |
<key>OpenAireDataSourceId</key> |
|
| 38 |
<value></value> |
|
| 39 |
</FIELD> |
|
| 40 |
<FIELD> |
|
| 41 |
<key>NamespacePrefix</key> |
|
| 42 |
<value></value> |
|
| 43 |
</FIELD> |
|
| 44 |
<FIELD> |
|
| 45 |
<key>VERIFIED</key> |
|
| 46 |
<value>NO</value> |
|
| 47 |
</FIELD> |
|
| 48 |
<FIELD> |
|
| 49 |
<key>aggregatorName</key> |
|
| 50 |
<value></value> |
|
| 51 |
</FIELD> |
|
| 52 |
<FIELD> |
|
| 53 |
<key>dateOfValidation</key> |
|
| 54 |
<value></value> |
|
| 55 |
</FIELD> |
|
| 56 |
<FIELD> |
|
| 57 |
<key>dateOfCollection</key> |
|
| 58 |
<value></value> |
|
| 59 |
</FIELD> |
|
| 60 |
<FIELD> |
|
| 61 |
<key>ACTID</key> |
|
| 62 |
<value></value> |
|
| 63 |
</FIELD> |
|
| 64 |
</EXTRA_FIELDS> |
|
| 65 |
<REGISTERED_BY></REGISTERED_BY> |
|
| 66 |
</CONFIGURATION> |
|
| 67 |
<STATUS> |
|
| 68 |
<NUMBER_OF_OBJECTS></NUMBER_OF_OBJECTS> |
|
| 69 |
<LAST_UPDATE value=""/> |
|
| 70 |
</STATUS> |
|
| 71 |
<QOS> |
|
| 72 |
<AVAILABILITY>0</AVAILABILITY> |
|
| 73 |
<CAPACITY/> |
|
| 74 |
<THROUGHPUT>0.0</THROUGHPUT> |
|
| 75 |
</QOS> |
|
| 76 |
<SECURITY_PARAMETERS/> |
|
| 77 |
<BLACKBOARD/> |
|
| 78 |
</BODY> |
|
| 79 |
</RESOURCE_PROFILE> |
|
| modules/dnet-openaire-exporter/branches/dsm/src/main/java/eu/dnetlib/openaire/exporter/datasource/clients/DatasourceDaoImpl.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.openaire.exporter.datasource.clients; |
| 2 | 2 |
|
| 3 |
import java.util.HashSet; |
|
| 3 | 4 |
import java.util.List; |
| 4 | 5 |
import java.util.Objects; |
| 6 |
import java.util.Set; |
|
| 5 | 7 |
import java.util.stream.Collectors; |
| 6 | 8 |
|
| 7 | 9 |
import com.google.common.collect.Lists; |
| ... | ... | |
| 9 | 11 |
import eu.dnetlib.enabling.datasources.common.DatasourceManagerException; |
| 10 | 12 |
import eu.dnetlib.openaire.exporter.datasource.repository.ApiDbEntryRepository; |
| 11 | 13 |
import eu.dnetlib.openaire.exporter.datasource.repository.CountryTermRepository; |
| 14 |
import eu.dnetlib.openaire.exporter.datasource.repository.DatasourceApiDbEntryRepository; |
|
| 12 | 15 |
import eu.dnetlib.openaire.exporter.datasource.repository.DatasourceDbEntryRepository; |
| 16 |
import eu.dnetlib.openaire.exporter.model.datasource.RequestFilter; |
|
| 17 |
import eu.dnetlib.openaire.exporter.model.datasource.RequestSort; |
|
| 18 |
import eu.dnetlib.openaire.exporter.model.datasource.RequestSortOrder; |
|
| 13 | 19 |
import eu.dnetlib.openaire.exporter.model.datasource.db.ApiDbEntry; |
| 20 |
import eu.dnetlib.openaire.exporter.model.datasource.db.DatasourceApiDbEntry; |
|
| 14 | 21 |
import eu.dnetlib.openaire.exporter.model.datasource.db.DatasourceDbEntry; |
| 15 | 22 |
import eu.dnetlib.openaire.exporter.vocabularies.Country; |
| 16 | 23 |
import eu.dnetlib.openaire.exporter.vocabularies.Vocabulary; |
| ... | ... | |
| 19 | 26 |
import org.apache.commons.logging.LogFactory; |
| 20 | 27 |
import org.apache.http.HttpStatus; |
| 21 | 28 |
import org.springframework.beans.factory.annotation.Autowired; |
| 29 |
import org.springframework.data.domain.Page; |
|
| 22 | 30 |
import org.springframework.data.domain.PageRequest; |
| 23 |
import org.springframework.data.domain.Slice;
|
|
| 31 |
import org.springframework.data.jpa.domain.Specification;
|
|
| 24 | 32 |
import org.springframework.stereotype.Component; |
| 25 |
import org.springframework.util.concurrent.ListenableFuture; |
|
| 26 | 33 |
|
| 34 |
import static eu.dnetlib.openaire.exporter.datasource.repository.DatasourceSpecs.apiSpec; |
|
| 35 |
import static eu.dnetlib.openaire.exporter.datasource.repository.DatasourceSpecs.dsSpec; |
|
| 36 |
|
|
| 27 | 37 |
/** |
| 28 | 38 |
* Created by claudio on 20/10/2016. |
| 29 | 39 |
*/ |
| ... | ... | |
| 45 | 55 |
private ApiDbEntryRepository apiRepository; |
| 46 | 56 |
|
| 47 | 57 |
@Autowired |
| 58 |
private DatasourceApiDbEntryRepository dsApiRepository; |
|
| 59 |
|
|
| 60 |
@Autowired |
|
| 48 | 61 |
private VocabularyClient vocabularyClient; |
| 49 | 62 |
|
| 50 |
|
|
| 51 | 63 |
@Override |
| 52 | 64 |
public List<Country> listCountries() throws DatasourceManagerException {
|
| 53 | 65 |
final List<Country> countries = Lists.newArrayList(); |
| ... | ... | |
| 70 | 82 |
} |
| 71 | 83 |
|
| 72 | 84 |
@Override |
| 73 |
public ListenableFuture<Slice<DatasourceDbEntry>> searchByName(final String name, final int page, final int size) {
|
|
| 74 |
log.debug(String.format("search ds by name '%s'", name));
|
|
| 75 |
return dsRepository.findDistinctByOfficialnameContainingOrEnglishnameContainingAllIgnoreCase( |
|
| 76 |
name, |
|
| 77 |
name, |
|
| 78 |
new PageRequest(page, size)); |
|
| 79 |
} |
|
| 85 |
public Page<DatasourceDbEntry> search(final RequestSort requestSortBy, RequestSortOrder order, RequestFilter requestFilter, final int page, final int size) |
|
| 86 |
throws DatasourceManagerException {
|
|
| 80 | 87 |
|
| 81 |
@Override |
|
| 82 |
public ListenableFuture<Slice<DatasourceDbEntry>> searchByCountry(final String country, final int page, final int size) {
|
|
| 83 |
log.debug(String.format("search ds by country '%s'", country));
|
|
| 84 |
return dsRepository.findDistinctByOrganizationsCountryIgnoreCase(country, new PageRequest(page, size)); |
|
| 88 |
final Specification<DatasourceDbEntry> spec = dsSpec(requestSortBy, order, requestFilter); |
|
| 89 |
return dsRepository.findAll(spec, new PageRequest(page, size)); |
|
| 85 | 90 |
} |
| 86 | 91 |
|
| 87 | 92 |
@Override |
| 88 |
public ListenableFuture<Slice<DatasourceDbEntry>> searchByContactemail(final String email, final int page, final int size) {
|
|
| 89 |
log.debug(String.format("search ds by email '%s'", email));
|
|
| 90 |
return dsRepository.findDistinctByContactemailContainingAllIgnoreCase(email, new PageRequest(page, size)); |
|
| 91 |
} |
|
| 92 |
|
|
| 93 |
@Override |
|
| 94 |
public ListenableFuture<Slice<DatasourceDbEntry>> searchByRegisteringUser(final String registeringUser, final int page, final int size) |
|
| 95 |
throws DatasourceManagerException {
|
|
| 96 |
log.debug(String.format("search ds by registering user '%s'", registeringUser));
|
|
| 97 |
return dsRepository.findDistinctByRegisteredbyIgnoreCase(registeringUser, new PageRequest(page, size)); |
|
| 98 |
} |
|
| 99 |
|
|
| 100 |
@Override |
|
| 101 | 93 |
public DatasourceDbEntry getDs(final String dsId) throws DatasourceManagerException {
|
| 102 | 94 |
return dsRepository.findOne(dsId); |
| 103 | 95 |
} |
| ... | ... | |
| 125 | 117 |
} |
| 126 | 118 |
|
| 127 | 119 |
@Override |
| 128 |
public void deleteApi(final String dsId, final String apiId) {
|
|
| 120 |
public void deleteApi(final String dsId, final String apiId) throws DatasourceManagerException {
|
|
| 121 |
final ApiDbEntry api = apiRepository.findOne(apiId); |
|
| 122 |
if (!api.getRemovable()) {
|
|
| 123 |
throw new DatasourceManagerException(HttpStatus.SC_UNAUTHORIZED, "api is not removable"); |
|
| 124 |
} |
|
| 125 |
|
|
| 129 | 126 |
apiRepository.delete(apiId); |
| 130 | 127 |
log.info(String.format("deleted api '%s'", apiId));
|
| 131 | 128 |
} |
| 132 | 129 |
|
| 133 | 130 |
@Override |
| 134 | 131 |
public void addApi(final ApiDbEntry api) {
|
| 135 |
|
|
| 136 | 132 |
if (StringUtils.isBlank(api.getId())) {
|
| 137 | 133 |
api.setId(ApiDbEntryRepository.createId(api)); |
| 138 | 134 |
log.info(String.format("missing api id, created '%s'"));
|
| ... | ... | |
| 178 | 174 |
} |
| 179 | 175 |
|
| 180 | 176 |
@Override |
| 177 |
public List<String> findBaseURLs(final RequestFilter requestFilter, final int page, final int size) throws DatasourceManagerException {
|
|
| 178 |
final PageRequest pageable = new PageRequest(page, size); |
|
| 179 |
final Specification<DatasourceApiDbEntry> spec = apiSpec(requestFilter); |
|
| 180 |
final Set<String> set = dsApiRepository.findAll(spec, pageable).getContent().stream() |
|
| 181 |
.map(DatasourceApiDbEntry::getBaseurl) |
|
| 182 |
.filter(StringUtils::isNotBlank) |
|
| 183 |
.collect(Collectors.toCollection(HashSet::new)); |
|
| 184 |
return Lists.newArrayList(set); |
|
| 185 |
} |
|
| 186 |
|
|
| 187 |
@Override |
|
| 181 | 188 |
public void updateTimezone(final String dsId, final String timezone) {
|
| 182 | 189 |
dsRepository.setTimezone(dsId, timezone); |
| 183 | 190 |
log.info(String.format("updated datasource '%s' with timezone '%s'", dsId, timezone));
|
| modules/dnet-openaire-exporter/branches/dsm/src/main/java/eu/dnetlib/openaire/exporter/datasource/clients/ISLookupClient.java | ||
|---|---|---|
| 3 | 3 |
import java.io.IOException; |
| 4 | 4 |
import java.nio.charset.Charset; |
| 5 | 5 |
import java.util.List; |
| 6 |
import java.util.Map; |
|
| 6 | 7 |
import java.util.Queue; |
| 8 |
import java.util.concurrent.LinkedBlockingQueue; |
|
| 9 |
import java.util.function.Function; |
|
| 10 |
import java.util.stream.Collectors; |
|
| 7 | 11 |
|
| 8 | 12 |
import com.google.common.collect.Lists; |
| 13 |
import com.google.common.xml.XmlEscapers; |
|
| 9 | 14 |
import eu.dnetlib.OpenaireExporterConfig; |
| 15 |
import eu.dnetlib.enabling.datasources.common.DatasourceManagerException; |
|
| 16 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; |
|
| 10 | 17 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; |
| 11 | 18 |
import eu.dnetlib.openaire.exporter.datasource.clients.utils.IndexDsInfo; |
| 19 |
import eu.dnetlib.openaire.exporter.funders.context.Context; |
|
| 20 |
import eu.dnetlib.openaire.exporter.funders.context.MappingUtils; |
|
| 12 | 21 |
import org.apache.commons.io.IOUtils; |
| 22 |
import org.apache.commons.lang3.StringUtils; |
|
| 13 | 23 |
import org.apache.commons.logging.Log; |
| 14 | 24 |
import org.apache.commons.logging.LogFactory; |
| 15 | 25 |
import org.springframework.beans.factory.annotation.Autowired; |
| ... | ... | |
| 33 | 43 |
private ISLookUpService isLookUpService; |
| 34 | 44 |
|
| 35 | 45 |
@Cacheable("datasources-is-cache")
|
| 36 |
public IndexDsInfo calculateCurrentIndexDsInfo(final Queue<Throwable> errors) throws IOException {
|
|
| 46 |
public IndexDsInfo calculateCurrentIndexDsInfo() throws DatasourceManagerException {
|
|
| 37 | 47 |
log.warn("calculateCurrentIndexDsInfo(): not using cache");
|
| 38 |
final String[] arr = _isLookUp(_getQuery(config.getFindIndexDsInfo()), errors).split("@@@");
|
|
| 39 |
return new IndexDsInfo( |
|
| 40 |
_isLookUp(_getQuery(config.getFindSolrIndexUrl()), errors), |
|
| 48 |
final String[] arr; |
|
| 49 |
try {
|
|
| 50 |
arr = _isLookUp(_getQuery(config.getFindIndexDsInfo())).split("@@@");
|
|
| 51 |
return new IndexDsInfo( |
|
| 52 |
_isLookUp(_getQuery(config.getFindSolrIndexUrl())), |
|
| 41 | 53 |
arr[0].trim(), arr[1].trim(), arr[2].trim()); |
| 54 |
} catch (IOException | ISLookUpException e) {
|
|
| 55 |
throw new DatasourceManagerException("unable fetch index DS information from IS");
|
|
| 56 |
} |
|
| 42 | 57 |
} |
| 43 | 58 |
|
| 44 | 59 |
@Cacheable("datasources-is-cache")
|
| 45 | 60 |
public String getObjectStoreId(final String dsId, final Queue<Throwable> errors) throws IOException {
|
| 46 | 61 |
log.warn(String.format("getObjectStoreId(%s): not using cache", dsId));
|
| 47 | 62 |
final String xqueryTemplate = _getQuery(config.getFindObjectStore()); |
| 48 |
return _isLookUp(String.format(xqueryTemplate, dsId), errors); |
|
| 63 |
try {
|
|
| 64 |
return _isLookUp(String.format(xqueryTemplate, dsId)); |
|
| 65 |
} catch (ISLookUpException e) {
|
|
| 66 |
errors.add(new DatasourceManagerException("unable to find objectStore for datasource " + dsId));
|
|
| 67 |
return ""; |
|
| 68 |
} |
|
| 49 | 69 |
} |
| 50 | 70 |
|
| 51 |
public List<String> getContextProfiles(final Queue<Throwable> errors) throws IOException {
|
|
| 71 |
@Cacheable("datasources-is-cache")
|
|
| 72 |
public Map<String, Context> getContextMap() throws IOException {
|
|
| 73 |
final Queue<Throwable> errors = new LinkedBlockingQueue<>(); |
|
| 74 |
try {
|
|
| 75 |
return getContextProfiles(errors).stream() |
|
| 76 |
.filter(StringUtils::isNotBlank) |
|
| 77 |
.map(s -> MappingUtils.parseContext(s, errors)) |
|
| 78 |
.collect(Collectors.toMap( |
|
| 79 |
Context::getId, |
|
| 80 |
Function.identity())); |
|
| 81 |
} finally {
|
|
| 82 |
if (!errors.isEmpty()) {
|
|
| 83 |
log.error(errors); |
|
| 84 |
} |
|
| 85 |
} |
|
| 86 |
} |
|
| 87 |
|
|
| 88 |
private void updateDatasourceField(final String dsId, final String element, final String value, final Queue<Throwable> errors) {
|
|
| 89 |
try {
|
|
| 90 |
final String xquery = _getQuery(config.getUpdateRepoField()); |
|
| 91 |
_isLookUp( |
|
| 92 |
String.format( |
|
| 93 |
xquery, |
|
| 94 |
XmlEscapers.xmlAttributeEscaper().escape(dsId), |
|
| 95 |
XmlEscapers.xmlContentEscaper().escape(element), |
|
| 96 |
XmlEscapers.xmlContentEscaper().escape(value))); |
|
| 97 |
} catch (IOException | ISLookUpException e) {
|
|
| 98 |
errors.add(e); |
|
| 99 |
} |
|
| 100 |
} |
|
| 101 |
|
|
| 102 |
/// HELPERS |
|
| 103 |
|
|
| 104 |
private List<String> getContextProfiles(final Queue<Throwable> errors) throws IOException {
|
|
| 52 | 105 |
log.warn("getContextProfiles(): not using cache");
|
| 53 | 106 |
final String xqueryTemplate = _getQuery(config.getFindFunderContexts()); |
| 54 | 107 |
return _quickSeachProfile(xqueryTemplate, errors); |
| ... | ... | |
| 58 | 111 |
return IOUtils.toString(resource.getInputStream(), Charset.defaultCharset()); |
| 59 | 112 |
} |
| 60 | 113 |
|
| 61 |
private String _isLookUp(final String xquery, final Queue<Throwable> errors) {
|
|
| 62 |
try {
|
|
| 63 |
log.debug(String.format("running xquery: %s", xquery));
|
|
| 64 |
final String res = isLookUpService.getResourceProfileByQuery(xquery); |
|
| 65 |
log.debug(String.format("query result: %s", res));
|
|
| 66 |
return res; |
|
| 67 |
} catch (Throwable e) {
|
|
| 68 |
errors.add(e); |
|
| 69 |
return ""; |
|
| 70 |
} |
|
| 114 |
private String _isLookUp(final String xquery) throws ISLookUpException {
|
|
| 115 |
|
|
| 116 |
log.debug(String.format("running xquery: %s", xquery));
|
|
| 117 |
final String res = isLookUpService.getResourceProfileByQuery(xquery); |
|
| 118 |
log.debug(String.format("query result: %s", res));
|
|
| 119 |
return res; |
|
| 71 | 120 |
} |
| 72 | 121 |
|
| 73 | 122 |
private List<String> _quickSeachProfile(final String xquery, final Queue<Throwable> errors) {
|
| modules/dnet-openaire-exporter/branches/dsm/src/main/java/eu/dnetlib/openaire/exporter/datasource/DatasourceManagerCore.java | ||
|---|---|---|
| 2 | 2 |
|
| 3 | 3 |
import java.util.List; |
| 4 | 4 |
import java.util.Queue; |
| 5 |
import java.util.concurrent.*; |
|
| 5 |
import java.util.concurrent.CountDownLatch; |
|
| 6 |
import java.util.concurrent.ScheduledThreadPoolExecutor; |
|
| 7 |
import java.util.concurrent.TimeUnit; |
|
| 8 |
import java.util.concurrent.TimeoutException; |
|
| 6 | 9 |
import java.util.stream.Collectors; |
| 7 |
import javax.annotation.Nullable; |
|
| 8 | 10 |
import javax.annotation.PostConstruct; |
| 9 | 11 |
|
| 10 | 12 |
import com.google.common.collect.Lists; |
| 13 |
import com.google.common.collect.Queues; |
|
| 11 | 14 |
import com.google.common.util.concurrent.*; |
| 12 | 15 |
import eu.dnetlib.OpenaireExporterConfig; |
| 13 | 16 |
import eu.dnetlib.enabling.datasources.common.AggregationInfo; |
| ... | ... | |
| 16 | 19 |
import eu.dnetlib.openaire.exporter.datasource.clients.*; |
| 17 | 20 |
import eu.dnetlib.openaire.exporter.datasource.clients.utils.IndexDsInfo; |
| 18 | 21 |
import eu.dnetlib.openaire.exporter.datasource.clients.utils.IndexRecordsInfo; |
| 22 |
import eu.dnetlib.openaire.exporter.model.ApiDetailsResponse; |
|
| 19 | 23 |
import eu.dnetlib.openaire.exporter.model.ConversionUtils; |
| 20 |
import eu.dnetlib.openaire.exporter.model.datasource.ApiDetails; |
|
| 21 |
import eu.dnetlib.openaire.exporter.model.datasource.DatasourceDetails; |
|
| 22 |
import eu.dnetlib.openaire.exporter.model.datasource.DatasourceInfo; |
|
| 24 |
import eu.dnetlib.openaire.exporter.model.DatasourceResponse; |
|
| 25 |
import eu.dnetlib.openaire.exporter.model.datasource.*; |
|
| 23 | 26 |
import eu.dnetlib.openaire.exporter.model.datasource.db.ApiDbEntry; |
| 24 | 27 |
import eu.dnetlib.openaire.exporter.model.datasource.db.DatasourceDbEntry; |
| 25 | 28 |
import eu.dnetlib.openaire.exporter.vocabularies.Country; |
| ... | ... | |
| 27 | 30 |
import org.apache.commons.logging.Log; |
| 28 | 31 |
import org.apache.commons.logging.LogFactory; |
| 29 | 32 |
import org.springframework.beans.factory.annotation.Autowired; |
| 30 |
import org.springframework.data.domain.Slice;
|
|
| 33 |
import org.springframework.data.domain.Page;
|
|
| 31 | 34 |
import org.springframework.stereotype.Component; |
| 32 |
import org.springframework.util.concurrent.ListenableFuture; |
|
| 33 |
import org.springframework.util.concurrent.ListenableFutureCallback; |
|
| 34 | 35 |
|
| 36 |
import static eu.dnetlib.openaire.exporter.datasource.clients.ResponseUtils.apiResponse; |
|
| 37 |
import static eu.dnetlib.openaire.exporter.datasource.clients.ResponseUtils.datasourceResponse; |
|
| 38 |
|
|
| 35 | 39 |
@Component |
| 36 | 40 |
public class DatasourceManagerCore {
|
| 37 | 41 |
|
| ... | ... | |
| 75 | 79 |
return dsDao.listIds(page, size); |
| 76 | 80 |
} |
| 77 | 81 |
|
| 78 |
public List<DatasourceInfo> searchByName(final String name, int page, int size) throws DatasourceManagerException {
|
|
| 79 |
log.debug(String.format("search ds by name '%s'", name));
|
|
| 80 |
return _search(dsDao.searchByName(name, page, size)); |
|
| 81 |
} |
|
| 82 |
public DatasourceResponse search(RequestSort requestSortBy, RequestSortOrder order, RequestFilter requestFilter, int page, int size) |
|
| 83 |
throws DatasourceManagerException {
|
|
| 84 |
final List<DatasourceInfo> datasourceInfo = Lists.newArrayList(); |
|
| 85 |
final Queue<Throwable> errors = Queues.newLinkedBlockingQueue(); |
|
| 86 |
final CountDownLatch outerLatch = new CountDownLatch(2); |
|
| 82 | 87 |
|
| 83 |
public List<DatasourceInfo> searchByContactemail(final String email, int page, int size) throws DatasourceManagerException {
|
|
| 84 |
log.debug(String.format("search ds by Contactemail '%s'", email));
|
|
| 85 |
return _search(dsDao.searchByContactemail(email, page, size)); |
|
| 86 |
} |
|
| 88 |
final Page<DatasourceDbEntry> dsPage = dsDao.search(requestSortBy, order, requestFilter, page, size); |
|
| 89 |
if (dsPage.getTotalElements() > 0 && dsPage.getNumberOfElements() > 0) {
|
|
| 90 |
dsPage.forEach(d -> datasourceInfo.add(enrichDatasourceInfo(ConversionUtils.asDetails(d), outerLatch, errors))); |
|
| 91 |
waitLatch(outerLatch, errors, config.getRequestTimeout()); |
|
| 92 |
} |
|
| 87 | 93 |
|
| 88 |
public List<DatasourceInfo> searchByCountry(final String country, int page, int size) throws DatasourceManagerException {
|
|
| 89 |
log.debug(String.format("search ds by country '%s'", country));
|
|
| 90 |
return _search(dsDao.searchByCountry(country, page, size)); |
|
| 94 |
return datasourceResponse(datasourceInfo, errors, dsPage.getTotalElements()); |
|
| 91 | 95 |
} |
| 92 | 96 |
|
| 93 |
public List<DatasourceInfo> searchByRegisteringUser(final String registeredBy, final int page, final int size) throws DatasourceManagerException {
|
|
| 94 |
log.debug(String.format("search ds by registering user '%s'", registeredBy));
|
|
| 95 |
return _search(dsDao.searchByRegisteringUser(registeredBy, page, size)); |
|
| 97 |
public List<String> findBaseURLs(final RequestFilter requestFilter, final int page, final int size) throws DatasourceManagerException {
|
|
| 98 |
return dsDao.findBaseURLs(requestFilter, page, size); |
|
| 96 | 99 |
} |
| 97 | 100 |
|
| 98 |
public DatasourceInfo getDsInfoById(final String dsId) throws DatasourceManagerException {
|
|
| 99 |
|
|
| 100 |
final CountDownLatch outerLatch = new CountDownLatch(3); |
|
| 101 |
final Queue<Throwable> errors = new ConcurrentLinkedQueue<>(); |
|
| 102 |
final DatasourceInfo datasourceInfo = new DatasourceInfo(); |
|
| 103 |
|
|
| 104 |
getAggregationHistory(dsId, outerLatch, errors, datasourceInfo); |
|
| 105 |
|
|
| 106 |
Futures.addCallback( |
|
| 107 |
executor.submit(() -> (DatasourceDbEntry) dsDao.getDs(dsId)), |
|
| 108 |
new FutureCallback<DatasourceDbEntry>() {
|
|
| 109 |
@Override |
|
| 110 |
public void onSuccess(@Nullable final DatasourceDbEntry d) {
|
|
| 111 |
datasourceInfo.setDatasourceDetails(ConversionUtils.asDetails(d)); |
|
| 112 |
} |
|
| 113 |
|
|
| 114 |
@Override |
|
| 115 |
public void onFailure(final Throwable e) {
|
|
| 116 |
errors.add(e); |
|
| 117 |
outerLatch.countDown(); |
|
| 118 |
} |
|
| 119 |
}, |
|
| 120 |
executor); |
|
| 121 |
|
|
| 122 |
getIndexDsInfo(dsId, outerLatch, errors, datasourceInfo); |
|
| 123 |
waitLatch(outerLatch, errors, config.getRequestTimeout()); |
|
| 124 |
|
|
| 125 |
if (!errors.isEmpty()) {
|
|
| 126 |
errors.forEach(e -> {
|
|
| 127 |
//TODO improve error reporting |
|
| 128 |
log.error(ExceptionUtils.getRootCauseMessage(e)); |
|
| 129 |
}); |
|
| 130 |
} |
|
| 131 |
|
|
| 132 |
return datasourceInfo; |
|
| 133 |
} |
|
| 134 |
|
|
| 135 |
public List<ApiDetails> getApis(final String dsId) throws DatasourceManagerException {
|
|
| 101 |
public ApiDetailsResponse getApis(final String dsId) throws DatasourceManagerException {
|
|
| 136 | 102 |
final List<ApiDbEntry> apis = dsDao.getApis(dsId); |
| 137 |
return apis.stream()
|
|
| 103 |
final List<ApiDetails> api = apis.stream()
|
|
| 138 | 104 |
.map(ConversionUtils::asDetails) |
| 139 | 105 |
.collect(Collectors.toList()); |
| 106 |
return apiResponse(api, api.size()); |
|
| 140 | 107 |
} |
| 141 | 108 |
|
| 142 | 109 |
public void setManaged(final String dsId, final boolean managed) throws DatasourceManagerException {
|
| ... | ... | |
| 154 | 121 |
|
| 155 | 122 |
public void save(final DatasourceDetails d) throws DatasourceManagerException {
|
| 156 | 123 |
dsDao.saveDs(ConversionUtils.asDbEntry(d)); |
| 124 |
|
|
| 125 |
//TODO add also to the IS |
|
| 157 | 126 |
} |
| 158 | 127 |
|
| 159 | 128 |
public void updateDatasourcename(final String dsId, final String officialname, final String englishname) throws DatasourceManagerException {
|
| 160 | 129 |
log.info(String.format("updated datasource '%s' with officialname '%s' and englishname '%s'", dsId, officialname, englishname));
|
| 161 | 130 |
dsDao.updateDatasourceName(dsId, officialname, englishname); |
| 131 |
|
|
| 132 |
//TODO update also to the IS |
|
| 162 | 133 |
} |
| 163 | 134 |
|
| 164 | 135 |
public void updateCoordinates(final String dsId, final Double latitude, final Double longitude) throws DatasourceManagerException {
|
| 165 | 136 |
log.info(String.format("updated datasource '%s' with coordinates Lat:'%s', Lon:'%s'", dsId, latitude, longitude));
|
| 166 | 137 |
dsDao.updateCoordinates(dsId, latitude, longitude); |
| 138 |
|
|
| 139 |
//TODO update also to the IS |
|
| 167 | 140 |
} |
| 168 | 141 |
|
| 169 | 142 |
public void updateApiBaseurl(final String apiId, final String baseUrl) throws DatasourceManagerException {
|
| 170 | 143 |
log.info(String.format("updated api '%s' baseurl with '%s'", apiId, baseUrl));
|
| 171 | 144 |
dsDao.updateBaseUrl(apiId, baseUrl); |
| 145 |
|
|
| 146 |
//TODO update also to the IS |
|
| 172 | 147 |
} |
| 173 | 148 |
|
| 174 | 149 |
public void updateTimezone(final String dsId, final String timezone) throws DatasourceManagerException {
|
| 175 | 150 |
log.info(String.format("updated datasource '%s' timezone with '%s'", dsId, timezone));
|
| 176 | 151 |
dsDao.updateTimezone(dsId, timezone); |
| 152 |
|
|
| 153 |
//TODO update also to the IS |
|
| 177 | 154 |
} |
| 178 | 155 |
|
| 179 | 156 |
public void updateDsTypology(final String dsId, final String typology) throws DatasourceManagerException {
|
| 180 | 157 |
log.info(String.format("updated datasource '%s' typology with '%s'", dsId, typology));
|
| 181 | 158 |
dsDao.updateTypology(dsId, typology); |
| 159 |
|
|
| 160 |
//TODO update also to the IS |
|
| 182 | 161 |
} |
| 183 | 162 |
|
| 184 | 163 |
public void updateApiCompatibility(final String apiId, final String compliance, final boolean override) throws DatasourceManagerException {
|
| 185 | 164 |
log.info(String.format("updated api '%s' compliance with '%s'", apiId, compliance));
|
| 186 | 165 |
dsDao.updateCompliance(null, apiId, compliance, override); |
| 166 |
|
|
| 167 |
//TODO update also to the IS |
|
| 187 | 168 |
} |
| 188 | 169 |
|
| 189 | 170 |
public void deleteDs(final String dsId) throws DatasourceManagerException {
|
| ... | ... | |
| 191 | 172 |
dsDao.deleteDs(dsId); |
| 192 | 173 |
} |
| 193 | 174 |
|
| 175 |
public void deleteApi(final String apiId) throws DatasourceManagerException {
|
|
| 176 |
dsDao.deleteApi(null, apiId); |
|
| 177 |
} |
|
| 178 |
|
|
| 194 | 179 |
public void dropCaches() {
|
| 195 | 180 |
mongoLoggerClient.dropCache(); |
| 196 | 181 |
isLookupClient.dropCache(); |
| 197 | 182 |
vocabularyClient.dropCache(); |
| 198 | 183 |
} |
| 199 | 184 |
|
| 200 |
//////////////// |
|
| 185 |
// HELPERS //////////////
|
|
| 201 | 186 |
|
| 202 |
private List<DatasourceInfo> _search(final ListenableFuture future) throws DatasourceManagerException {
|
|
| 203 |
final CountDownLatch outerLatch = new CountDownLatch(3); |
|
| 204 |
final List<DatasourceInfo> datasourceInfo = Lists.newArrayList(); |
|
| 205 |
final Queue<Throwable> errors = new ConcurrentLinkedQueue<>(); |
|
| 206 |
|
|
| 207 |
future.addCallback(new ListenableFutureCallback<Slice<DatasourceDbEntry>>() {
|
|
| 208 |
|
|
| 209 |
@Override |
|
| 210 |
public void onSuccess(final Slice<DatasourceDbEntry> datasources) {
|
|
| 211 |
datasources |
|
| 212 |
.map(ConversionUtils::asDetails) |
|
| 213 |
.forEach(d -> {
|
|
| 214 |
final DatasourceInfo info = new DatasourceInfo().setDatasourceDetails(d); |
|
| 215 |
getAggregationHistory(d.getId(), outerLatch, errors, info); |
|
| 216 |
getIndexDsInfo(d.getId(), outerLatch, errors, info); |
|
| 217 |
datasourceInfo.add(info); |
|
| 218 |
}); |
|
| 219 |
outerLatch.countDown(); |
|
| 220 |
} |
|
| 221 |
|
|
| 222 |
@Override |
|
| 223 |
public void onFailure(final Throwable e) {
|
|
| 224 |
errors.offer(e); |
|
| 225 |
outerLatch.countDown(); |
|
| 226 |
} |
|
| 227 |
}); |
|
| 228 |
|
|
| 229 |
waitLatch(outerLatch, errors, config.getRequestTimeout()); |
|
| 230 |
|
|
| 231 |
return datasourceInfo; |
|
| 187 |
private DatasourceInfo enrichDatasourceInfo(final DatasourceDetails d, final CountDownLatch outerLatch, final Queue<Throwable> errors) {
|
|
| 188 |
final DatasourceInfo dsInfo = new DatasourceInfo().setDatasource(d); |
|
| 189 |
getAggregationHistory(d.getId(), outerLatch, errors, dsInfo); |
|
| 190 |
getIndexDsInfo(d.getId(), outerLatch, errors, dsInfo); |
|
| 191 |
return dsInfo; |
|
| 232 | 192 |
} |
| 233 | 193 |
|
| 234 | 194 |
private void getAggregationHistory(final String dsId, |
| ... | ... | |
| 252 | 212 |
}, executor); |
| 253 | 213 |
} |
| 254 | 214 |
|
| 255 |
private void setAggregationHistory(DatasourceInfo datasourceInfo, final List<AggregationInfo> info) {
|
|
| 215 |
private void setAggregationHistory(final DatasourceInfo datasourceInfo, final List<AggregationInfo> info) {
|
|
| 256 | 216 |
datasourceInfo.setAggregationHistory(info); |
| 257 | 217 |
if (!info.isEmpty()) {
|
| 258 | 218 |
datasourceInfo |
| ... | ... | |
| 266 | 226 |
final Queue<Throwable> errors, |
| 267 | 227 |
final DatasourceInfo datasourceInfo) {
|
| 268 | 228 |
Futures.addCallback( |
| 269 |
executor.submit(() -> isLookupClient.calculateCurrentIndexDsInfo(errors)),
|
|
| 229 |
executor.submit(() -> isLookupClient.calculateCurrentIndexDsInfo()), |
|
| 270 | 230 |
new FutureCallback<IndexDsInfo>() {
|
| 271 | 231 |
|
| 272 | 232 |
public void onSuccess(final IndexDsInfo info) {
|
| ... | ... | |
| 316 | 276 |
} |
| 317 | 277 |
|
| 318 | 278 |
public void onFailure(final Throwable e) {
|
| 319 |
log.error(ExceptionUtils.getStackTrace(e)); |
|
| 279 |
//log.error(ExceptionUtils.getStackTrace(e));
|
|
| 320 | 280 |
errors.offer(e); |
| 321 | 281 |
outerLatch.countDown(); |
| 322 | 282 |
} |
| ... | ... | |
| 333 | 293 |
} |
| 334 | 294 |
} |
| 335 | 295 |
|
| 336 |
|
|
| 337 | 296 |
} |
| modules/dnet-openaire-exporter/branches/dsm/src/main/resources/application.properties | ||
|---|---|---|
| 45 | 45 |
openaire.exporter.findIndexDsInfo = /eu/dnetlib/openaire/exporter/xquery/findIndexDsInfo.xquery |
| 46 | 46 |
openaire.exporter.findObjectStore = /eu/dnetlib/openaire/exporter/xquery/findObjectStore.xquery |
| 47 | 47 |
openaire.exporter.findFunderContexts = /eu/dnetlib/openaire/exporter/xquery/findFunderContexts.xquery |
| 48 |
openaire.exporter.updateRepoField = /eu/dnetlib/openaire/exporter/xquery/updateRepoField.xquery |
|
| 48 | 49 |
|
| 49 | 50 |
# REST API CONFIGURATION |
| 50 |
openaire.exporter.swagger.apiTitle = OpenAIRE aggregator REST API |
|
| 51 |
openaire.exporter.swagger.apiDescription = The OpenAIRE data provision REST API allows developers to access the metadata information space of OpenAIRE programmatically. |
|
| 52 |
openaire.exporter.swagger.apiLicense = LICENSED UNDER GNU AFFERO GENERAL PUBLIC LICENSE. |
|
| 53 |
openaire.exporter.swagger.apiLicenseUrl = https://www.gnu.org/licenses/agpl-3.0.txt |
|
| 54 |
openaire.exporter.swagger.apiContacName = D-Net team |
|
| 55 |
openaire.exporter.swagger.apiContactUrl = http://www.openaire.eu |
|
| 56 |
openaire.exporter.swagger.apiContactEmail = dnet-team@isti.cnr.it |
|
| 51 |
openaire.exporter.swaggerDsm.apiTitle = OpenAIRE aggregator REST API
|
|
| 52 |
openaire.exporter.swaggerDsm.apiDescription = The OpenAIRE data provision REST API allows developers to access the metadata information space of OpenAIRE programmatically.
|
|
| 53 |
openaire.exporter.swaggerDsm.apiLicense = LICENSED UNDER GNU AFFERO GENERAL PUBLIC LICENSE.
|
|
| 54 |
openaire.exporter.swaggerDsm.apiLicenseUrl = https://www.gnu.org/licenses/agpl-3.0.txt
|
|
| 55 |
openaire.exporter.swaggerDsm.apiContacName = D-Net team
|
|
| 56 |
openaire.exporter.swaggerDsm.apiContactUrl = http://www.openaire.eu
|
|
| 57 |
openaire.exporter.swaggerDsm.apiContactEmail = dnet-team@isti.cnr.it
|
|
| 57 | 58 |
|
| 59 |
openaire.exporter.swaggerProjects.apiTitle = OpenAIRE projects REST API |
|
| 60 |
openaire.exporter.swaggerProjects.apiDescription = The OpenAIRE projects REST API allows programmatic access to funded research projects metadata. |
|
| 61 |
openaire.exporter.swaggerProjects.apiLicense = ${openaire.exporter.swaggerDsm.apiLicense}
|
|
| 62 |
openaire.exporter.swaggerProjects.apiLicenseUrl = ${openaire.exporter.swaggerDsm.apiLicenseUrl}
|
|
| 63 |
openaire.exporter.swaggerProjects.apiContacName = ${openaire.exporter.swaggerDsm.apiContacName}
|
|
| 64 |
openaire.exporter.swaggerProjects.apiContactUrl = ${openaire.exporter.swaggerDsm.apiContactUrl}
|
|
| 65 |
openaire.exporter.swaggerProjects.apiContactEmail = ${openaire.exporter.swaggerDsm.apiContactEmail}
|
|
| 66 |
|
|
| 67 |
openaire.exporter.swaggerFunders.apiTitle = OpenAIRE funders REST API |
|
| 68 |
openaire.exporter.swaggerFunders.apiDescription = The OpenAIRE funders REST API allows programmatic access to the funding agencies metadata in OpenAIRE. |
|
| 69 |
openaire.exporter.swaggerFunders.apiLicense = ${openaire.exporter.swaggerDsm.apiLicense}
|
|
| 70 |
openaire.exporter.swaggerFunders.apiLicenseUrl = ${openaire.exporter.swaggerDsm.apiLicenseUrl}
|
|
| 71 |
openaire.exporter.swaggerFunders.apiContacName = ${openaire.exporter.swaggerDsm.apiContacName}
|
|
| 72 |
openaire.exporter.swaggerFunders.apiContactUrl = ${openaire.exporter.swaggerDsm.apiContactUrl}
|
|
| 73 |
openaire.exporter.swaggerFunders.apiContactEmail = ${openaire.exporter.swaggerDsm.apiContactEmail}
|
|
| 74 |
|
|
| 58 | 75 |
# SWAGGER |
| 59 | 76 |
springfox.documentation.swagger.v2.path = ${server.contextPath}/api-docs
|
| 60 | 77 |
spring.jackson.date-format = eu.dnetlib.swagger.configuration.RFC3339DateFormat |
| modules/dnet-openaire-exporter/branches/dsm/src/main/java/eu/dnetlib/swagger/configuration/SwaggerDocumentationConfig.java | ||
|---|---|---|
| 2 | 2 |
|
| 3 | 3 |
import eu.dnetlib.OpenaireExporterConfig; |
| 4 | 4 |
import eu.dnetlib.OpenaireExporterConfig.Swagger; |
| 5 |
import eu.dnetlib.common.rmi.DNetRestDocumentation; |
|
| 6 | 5 |
import org.springframework.beans.factory.annotation.Autowired; |
| 7 | 6 |
import org.springframework.context.annotation.Bean; |
| 8 | 7 |
import org.springframework.context.annotation.Configuration; |
| 9 | 8 |
import springfox.documentation.builders.ApiInfoBuilder; |
| 10 |
import springfox.documentation.builders.RequestHandlerSelectors; |
|
| 11 | 9 |
import springfox.documentation.service.ApiInfo; |
| 12 | 10 |
import springfox.documentation.service.Contact; |
| 13 | 11 |
import springfox.documentation.spi.DocumentationType; |
Also available in: Unified diff
huge update: swagger documentation, alignment of methods with the service trunk