Revision 49534
Added by Claudio Atzori about 6 years ago
| modules/dnet-openaire-exporter/trunk/dnet-openaire-exporter.iml | ||
|---|---|---|
| 5 | 5 |
<configuration /> |
| 6 | 6 |
</facet> |
| 7 | 7 |
</component> |
| 8 |
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8" inherit-compiler-output="false">
|
|
| 8 |
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8"> |
|
| 9 | 9 |
<output url="file://$MODULE_DIR$/target/classes" /> |
| 10 | 10 |
<output-test url="file://$MODULE_DIR$/target/test-classes" /> |
| 11 | 11 |
<content url="file://$MODULE_DIR$"> |
| ... | ... | |
| 17 | 17 |
</content> |
| 18 | 18 |
<orderEntry type="inheritedJdk" /> |
| 19 | 19 |
<orderEntry type="sourceFolder" forTests="false" /> |
| 20 |
<orderEntry type="library" name="Maven: eu.dnetlib:cnr-rmi-api:2.6.1" level="project" /> |
|
| 21 |
<orderEntry type="library" name="Maven: eu.dnetlib:cnr-service-common:2.1.6" level="project" /> |
|
| 22 |
<orderEntry type="library" name="Maven: eu.dnetlib:cnr-misc-utils:1.0.4" level="project" /> |
|
| 23 |
<orderEntry type="library" name="Maven: eu.dnetlib:cnr-service-utils:1.0.2" level="project" /> |
|
| 24 |
<orderEntry type="library" name="Maven: eu.dnetlib:cnr-spring-utils:1.0.1" level="project" /> |
|
| 25 |
<orderEntry type="module" module-name="cnr-rmi-api" /> |
|
| 20 |
<orderEntry type="library" name="Maven: eu.dnetlib:dnet-pace-core:2.5.1" level="project" /> |
|
| 21 |
<orderEntry type="library" name="Maven: org.apache.solr:solr-solrj:5.5.4" level="project" /> |
|
| 22 |
<orderEntry type="library" name="Maven: eu.dnetlib:cnr-rmi-api:2.6.2-SNAPSHOT" level="project" /> |
|
| 26 | 23 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-core:3.1.5" level="project" /> |
| 27 |
<orderEntry type="library" name="Maven: org.codehaus.woodstox:woodstox-core-asl:4.4.1" level="project" /> |
|
| 28 |
<orderEntry type="library" name="Maven: org.codehaus.woodstox:stax2-api:3.1.4" level="project" /> |
|
| 29 | 24 |
<orderEntry type="library" name="Maven: org.apache.ws.xmlschema:xmlschema-core:2.2.1" level="project" /> |
| 30 | 25 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-frontend-jaxws:3.1.5" level="project" /> |
| 31 | 26 |
<orderEntry type="library" name="Maven: xml-resolver:xml-resolver:1.2" level="project" /> |
| ... | ... | |
| 43 | 38 |
<orderEntry type="library" name="Maven: org.apache.neethi:neethi:3.0.3" level="project" /> |
| 44 | 39 |
<orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" /> |
| 45 | 40 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-transports-http:3.1.5" level="project" /> |
| 46 |
<orderEntry type="module" module-name="cnr-service-common" />
|
|
| 41 |
<orderEntry type="library" name="Maven: eu.dnetlib:cnr-service-common:2.1.7-SNAPSHOT" level="project" />
|
|
| 47 | 42 |
<orderEntry type="library" name="Maven: org.quartz-scheduler:quartz:2.2.2" level="project" /> |
| 48 | 43 |
<orderEntry type="library" name="Maven: c3p0:c3p0:0.9.1.1" level="project" /> |
| 49 | 44 |
<orderEntry type="library" name="Maven: org.springframework:spring-tx:4.3.7.RELEASE" level="project" /> |
| 50 | 45 |
<orderEntry type="library" name="Maven: org.springframework:spring-beans:4.3.7.RELEASE" level="project" /> |
| 51 |
<orderEntry type="module" module-name="cnr-misc-utils" />
|
|
| 46 |
<orderEntry type="library" name="Maven: eu.dnetlib:cnr-misc-utils:1.0.5-SNAPSHOT" level="project" />
|
|
| 52 | 47 |
<orderEntry type="library" name="Maven: commons-lang:commons-lang:2.6" level="project" /> |
| 53 | 48 |
<orderEntry type="library" name="Maven: jaxen:jaxen:1.1.6" level="project" /> |
| 54 | 49 |
<orderEntry type="library" name="Maven: saxonica:saxon:9.1.0.8" level="project" /> |
| 55 | 50 |
<orderEntry type="library" name="Maven: saxonica:saxon-dom:9.1.0.8" level="project" /> |
| 56 | 51 |
<orderEntry type="library" name="Maven: jgrapht:jgrapht:0.7.2" level="project" /> |
| 57 | 52 |
<orderEntry type="library" name="Maven: net.sf.ehcache:ehcache:2.10.3" level="project" /> |
| 58 |
<orderEntry type="module" module-name="cnr-service-utils" />
|
|
| 53 |
<orderEntry type="library" name="Maven: eu.dnetlib:cnr-service-utils:1.0.3-SNAPSHOT" level="project" />
|
|
| 59 | 54 |
<orderEntry type="library" name="Maven: apache:oro:2.0.8" level="project" /> |
| 60 | 55 |
<orderEntry type="library" name="Maven: org.springframework:spring-context:4.3.7.RELEASE" level="project" /> |
| 61 |
<orderEntry type="module" module-name="cnr-spring-utils" />
|
|
| 56 |
<orderEntry type="library" name="Maven: eu.dnetlib:cnr-spring-utils:1.0.2-SNAPSHOT" level="project" />
|
|
| 62 | 57 |
<orderEntry type="library" name="Maven: org.springframework:spring-aop:4.3.7.RELEASE" level="project" /> |
| 63 | 58 |
<orderEntry type="library" name="Maven: jparsec:jparsec:2.0" level="project" /> |
| 64 | 59 |
<orderEntry type="library" name="Maven: runcc:runcc:0.7" level="project" /> |
| 65 | 60 |
<orderEntry type="module" module-name="dnet-openaireplus-mapping-utils" /> |
| 66 | 61 |
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.10" level="project" /> |
| 67 | 62 |
<orderEntry type="library" name="Maven: dom4j:dom4j:1.6.1" level="project" /> |
| 68 |
<orderEntry type="library" name="Maven: eu.dnetlib:dnet-openaire-data-protos:3.7.8" level="project" />
|
|
| 63 |
<orderEntry type="library" name="Maven: eu.dnetlib:dnet-openaire-data-protos:3.8.10" level="project" />
|
|
| 69 | 64 |
<orderEntry type="library" name="Maven: com.google.protobuf:protobuf-java:2.4.1" level="project" /> |
| 70 |
<orderEntry type="library" name="Maven: eu.dnetlib:dnet-pace-core:2.5.1" level="project" />
|
|
| 65 |
<orderEntry type="module" module-name="dnet-pace-core" />
|
|
| 71 | 66 |
<orderEntry type="library" name="Maven: edu.cmu:secondstring:1.0.0" level="project" /> |
| 72 | 67 |
<orderEntry type="library" name="Maven: commons-collections:commons-collections:3.2.2" level="project" /> |
| 73 | 68 |
<orderEntry type="library" name="Maven: eu.dnetlib:dnet-index-solr-common:1.3.1" level="project" /> |
| 74 |
<orderEntry type="library" name="Maven: org.apache.solr:solr-solrj:5.5.4" level="project" /> |
|
| 69 |
<orderEntry type="library" name="Maven: com.googlecode.protobuf-java-format:protobuf-java-format:1.2" level="project" /> |
|
| 70 |
<orderEntry type="library" name="Maven: org.apache.solr:solr-solrj:6.6.0" level="project" /> |
|
| 71 |
<orderEntry type="library" name="Maven: org.apache.commons:commons-math3:3.4.1" level="project" /> |
|
| 75 | 72 |
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.5.3" level="project" /> |
| 76 | 73 |
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.4.6" level="project" /> |
| 77 | 74 |
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpmime:4.5.3" level="project" /> |
| 78 |
<orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.4.6" level="project" /> |
|
| 75 |
<orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.4.10" level="project" /> |
|
| 76 |
<orderEntry type="library" name="Maven: org.codehaus.woodstox:stax2-api:3.1.4" level="project" /> |
|
| 77 |
<orderEntry type="library" name="Maven: org.codehaus.woodstox:woodstox-core-asl:4.4.1" level="project" /> |
|
| 79 | 78 |
<orderEntry type="library" name="Maven: org.noggit:noggit:0.6" level="project" /> |
| 80 |
<orderEntry type="library" name="Maven: com.googlecode.protobuf-java-format:protobuf-java-format:1.2" level="project" /> |
|
| 79 |
<orderEntry type="library" name="Maven: org.slf4j:jcl-over-slf4j:1.7.24" level="project" /> |
|
| 80 |
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.24" level="project" /> |
|
| 81 | 81 |
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-web:1.5.2.RELEASE" level="project" /> |
| 82 | 82 |
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter:1.5.2.RELEASE" level="project" /> |
| 83 | 83 |
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot:1.5.2.RELEASE" level="project" /> |
| ... | ... | |
| 116 | 116 |
<orderEntry type="library" name="Maven: org.springframework.data:spring-data-jpa:1.11.1.RELEASE" level="project" /> |
| 117 | 117 |
<orderEntry type="library" name="Maven: org.springframework.data:spring-data-commons:1.13.1.RELEASE" level="project" /> |
| 118 | 118 |
<orderEntry type="library" name="Maven: org.springframework:spring-orm:4.3.7.RELEASE" level="project" /> |
| 119 |
<orderEntry type="library" name="Maven: org.slf4j:jcl-over-slf4j:1.7.24" level="project" /> |
|
| 120 | 119 |
<orderEntry type="library" name="Maven: org.springframework:spring-aspects:4.3.7.RELEASE" level="project" /> |
| 121 | 120 |
<orderEntry type="library" name="Maven: org.postgresql:postgresql:42.0.0" level="project" /> |
| 122 | 121 |
<orderEntry type="library" name="Maven: org.apache.commons:commons-dbcp2:2.1" level="project" /> |
| ... | ... | |
| 145 | 144 |
<orderEntry type="library" name="Maven: io.springfox:springfox-spring-web:2.7.0" level="project" /> |
| 146 | 145 |
<orderEntry type="library" name="Maven: org.reflections:reflections:0.9.11" level="project" /> |
| 147 | 146 |
<orderEntry type="library" name="Maven: com.fasterxml:classmate:1.3.3" level="project" /> |
| 148 |
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.24" level="project" /> |
|
| 149 | 147 |
<orderEntry type="library" name="Maven: org.springframework.plugin:spring-plugin-core:1.2.0.RELEASE" level="project" /> |
| 150 | 148 |
<orderEntry type="library" name="Maven: org.springframework.plugin:spring-plugin-metadata:1.2.0.RELEASE" level="project" /> |
| 151 | 149 |
<orderEntry type="library" name="Maven: org.mapstruct:mapstruct:1.1.0.Final" level="project" /> |
| modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/model/project/ProjectTsv.java | ||
|---|---|---|
| 22 | 22 |
|
| 23 | 23 |
@Id |
| 24 | 24 |
@JsonIgnore |
| 25 |
private String id;
|
|
| 25 |
private long rowid;
|
|
| 26 | 26 |
private String code; |
| 27 | 27 |
private String acronym; |
| 28 | 28 |
private String title; |
| ... | ... | |
| 74 | 74 |
return StringUtils.isNotBlank(s) ? "\"" + s.replaceAll("\\n|\\t|\\s+", " ").replace("\"","\"\"").trim() + "\"" : "";
|
| 75 | 75 |
} |
| 76 | 76 |
|
| 77 |
public String getId() {
|
|
| 78 |
return id; |
|
| 77 |
public long getRowid() {
|
|
| 78 |
return rowid;
|
|
| 79 | 79 |
} |
| 80 | 80 |
|
| 81 |
public void setId(final String id) {
|
|
| 82 |
this.id = id;
|
|
| 81 |
public void setRowid(final long rowid) {
|
|
| 82 |
this.rowid = rowid;
|
|
| 83 | 83 |
} |
| 84 | 84 |
|
| 85 | 85 |
public String getCode() {
|
| ... | ... | |
| 217 | 217 |
public void setEmail(final String email) {
|
| 218 | 218 |
this.email = email; |
| 219 | 219 |
} |
| 220 |
|
|
| 220 | 221 |
} |
| modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/project/JdbcApiDao.java | ||
|---|---|---|
| 30 | 30 |
import org.apache.commons.logging.Log; |
| 31 | 31 |
import org.apache.commons.logging.LogFactory; |
| 32 | 32 |
import org.springframework.beans.factory.annotation.Autowired; |
| 33 |
import org.springframework.cache.annotation.CacheEvict; |
|
| 33 | 34 |
import org.springframework.cache.annotation.Cacheable; |
| 34 | 35 |
import org.springframework.stereotype.Component; |
| 35 | 36 |
|
| ... | ... | |
| 80 | 81 |
return res; |
| 81 | 82 |
} |
| 82 | 83 |
|
| 84 |
@CacheEvict(cacheNames = "fundingpath-ids", allEntries = true) |
|
| 85 |
public void dropCache() {
|
|
| 86 |
log.info("dropped project exporter fundingpath ids cache");
|
|
| 87 |
} |
|
| 88 |
|
|
| 83 | 89 |
public void processProjectDetails(final OutputStream outputStream, String format, Boolean compress) throws IOException {
|
| 84 | 90 |
final OutputStream out = getOutputStream(new BufferedOutputStream(outputStream), compress); |
| 85 | 91 |
try {
|
| modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/project/ProjectsController.java | ||
|---|---|---|
| 17 | 17 |
import eu.dnetlib.OpenaireExporterConfig.Project; |
| 18 | 18 |
import eu.dnetlib.common.rmi.DNetRestDocumentation; |
| 19 | 19 |
import eu.dnetlib.openaire.exporter.AbstractExporterController; |
| 20 |
import io.swagger.annotations.ApiOperation; |
|
| 21 |
import io.swagger.annotations.ApiResponse; |
|
| 22 |
import io.swagger.annotations.ApiResponses; |
|
| 20 | 23 |
import org.antlr.stringtemplate.StringTemplate; |
| 21 | 24 |
import org.apache.commons.io.IOUtils; |
| 22 | 25 |
import org.apache.commons.lang3.StringUtils; |
| ... | ... | |
| 133 | 136 |
dao.processProjectDetails(response.getOutputStream(), format, compress); |
| 134 | 137 |
} |
| 135 | 138 |
|
| 139 |
@ApiOperation(value = "drop the project exporter caches", notes = "drop the project exporter caches", httpMethod = "GET") |
|
| 140 |
@ApiResponses(value = {
|
|
| 141 |
@ApiResponse(code = 200, message = "OK"), |
|
| 142 |
@ApiResponse(code = 500, message = "unexpected error") }) |
|
| 143 |
@RequestMapping(value = "/export/dropcache.do", method = RequestMethod.GET) |
|
| 144 |
public void dropDsCaches() {
|
|
| 145 |
dao.dropCache(); |
|
| 146 |
} |
|
| 147 |
|
|
| 136 | 148 |
/** |
| 137 | 149 |
* Creates the query on the fundingProgramme specified in the given parameters. |
| 138 | 150 |
* |
| modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/datasource/clients/ISLookupClient.java | ||
|---|---|---|
| 11 | 11 |
import org.apache.commons.logging.Log; |
| 12 | 12 |
import org.apache.commons.logging.LogFactory; |
| 13 | 13 |
import org.springframework.beans.factory.annotation.Autowired; |
| 14 |
import org.springframework.cache.annotation.CacheEvict; |
|
| 14 | 15 |
import org.springframework.cache.annotation.Cacheable; |
| 15 | 16 |
import org.springframework.http.HttpStatus; |
| 16 | 17 |
import org.springframework.stereotype.Component; |
| ... | ... | |
| 51 | 52 |
} |
| 52 | 53 |
} |
| 53 | 54 |
|
| 55 |
@CacheEvict(cacheNames = "datasources-is-cache", allEntries = true) |
|
| 56 |
public void dropCache() {
|
|
| 57 |
log.info("dropped dsManager IS cache");
|
|
| 58 |
} |
|
| 59 |
|
|
| 54 | 60 |
} |
| modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/datasource/clients/VocabularyClient.java | ||
|---|---|---|
| 5 | 5 |
import org.apache.commons.logging.Log; |
| 6 | 6 |
import org.apache.commons.logging.LogFactory; |
| 7 | 7 |
import org.springframework.beans.factory.annotation.Autowired; |
| 8 |
import org.springframework.cache.annotation.CacheEvict; |
|
| 8 | 9 |
import org.springframework.cache.annotation.Cacheable; |
| 9 | 10 |
import org.springframework.http.ResponseEntity; |
| 10 | 11 |
import org.springframework.stereotype.Component; |
| ... | ... | |
| 32 | 33 |
return rsp.getBody(); |
| 33 | 34 |
} |
| 34 | 35 |
|
| 36 |
@CacheEvict(cacheNames = "vocabularies-cache", allEntries = true) |
|
| 37 |
public void dropCache() {
|
|
| 38 |
log.info("dropped dsManager vocabulary cache");
|
|
| 39 |
} |
|
| 40 |
|
|
| 35 | 41 |
} |
| modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/datasource/clients/DatasourceDao.java | ||
|---|---|---|
| 313 | 313 |
log.info(String.format("updated datasource '%s' with longitude '%s'", dsId, longitude));
|
| 314 | 314 |
} |
| 315 | 315 |
|
| 316 |
public void dropCaches() {
|
|
| 317 |
mongoLoggerClient.dropCache(); |
|
| 318 |
isLookupClient.dropCache(); |
|
| 319 |
vocabularyClient.dropCache(); |
|
| 320 |
} |
|
| 321 |
|
|
| 316 | 322 |
private void getIndexDsInfo(final String dsId, |
| 317 | 323 |
final CountDownLatch outerLatch, |
| 318 | 324 |
final Queue<Throwable> errors, |
| modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/datasource/clients/MongoLoggerClient.java | ||
|---|---|---|
| 23 | 23 |
import org.bson.Document; |
| 24 | 24 |
import org.bson.conversions.Bson; |
| 25 | 25 |
import org.springframework.beans.factory.annotation.Autowired; |
| 26 |
import org.springframework.cache.annotation.CacheEvict; |
|
| 26 | 27 |
import org.springframework.cache.annotation.Cacheable; |
| 27 | 28 |
import org.springframework.stereotype.Component; |
| 28 | 29 |
|
| ... | ... | |
| 75 | 76 |
.collect(Collectors.toList()); |
| 76 | 77 |
} |
| 77 | 78 |
|
| 79 |
@CacheEvict(cacheNames = "datasources-mongo-cache", allEntries = true) |
|
| 80 |
public void dropCache() {
|
|
| 81 |
log.info("dropped dsManager aggregation history cache");
|
|
| 82 |
} |
|
| 83 |
|
|
| 78 | 84 |
private Function<Document, AggregationInfo> getMapper() {
|
| 79 | 85 |
return new Function<Document, AggregationInfo>() {
|
| 80 | 86 |
|
| modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/datasource/DatasourcesApiController.java | ||
|---|---|---|
| 12 | 12 |
import eu.dnetlib.openaire.exporter.model.datasource.db.Datasource; |
| 13 | 13 |
import eu.dnetlib.openaire.exporter.model.datasource.db.SearchInterfacesEntry; |
| 14 | 14 |
import eu.dnetlib.openaire.exporter.vocabularies.Country; |
| 15 |
import io.swagger.annotations.ApiOperation; |
|
| 16 |
import io.swagger.annotations.ApiResponse; |
|
| 17 |
import io.swagger.annotations.ApiResponses; |
|
| 15 | 18 |
import org.apache.commons.lang.StringUtils; |
| 16 | 19 |
import org.apache.commons.logging.Log; |
| 17 | 20 |
import org.apache.commons.logging.LogFactory; |
| ... | ... | |
| 159 | 162 |
dsDao.updateLongitude(dsId, longitude); |
| 160 | 163 |
} |
| 161 | 164 |
|
| 165 |
@ApiOperation(value = "drop the datasource manager caches", notes = "drop the datasource manager caches", httpMethod = "GET") |
|
| 166 |
@ApiResponses(value = {
|
|
| 167 |
@ApiResponse(code = 200, message = "OK"), |
|
| 168 |
@ApiResponse(code = 500, message = "unexpected error") }) |
|
| 169 |
@RequestMapping(value = "/ds/dropcache.do", method = RequestMethod.GET) |
|
| 170 |
public void dropDsCaches() {
|
|
| 171 |
dsDao.dropCaches(); |
|
| 172 |
} |
|
| 173 |
|
|
| 174 |
|
|
| 162 | 175 |
} |
Also available in: Unified diff
added methods to drop the caches, changed project_tsv response, now providing all the organizations participating to a project