Revision 48481
Added by Alessia Bardi about 7 years ago
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/dnet-openaire-exporter.iml | ||
|---|---|---|
| 1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
| 2 |
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4"> |
|
| 3 |
<component name="FacetManager"> |
|
| 4 |
<facet type="Spring" name="Spring"> |
|
| 5 |
<configuration /> |
|
| 6 |
</facet> |
|
| 7 |
</component> |
|
| 8 |
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8"> |
|
| 9 |
<output url="file://$MODULE_DIR$/target/classes" /> |
|
| 10 |
<output-test url="file://$MODULE_DIR$/target/test-classes" /> |
|
| 11 |
<content url="file://$MODULE_DIR$"> |
|
| 12 |
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" /> |
|
| 13 |
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" /> |
|
| 14 |
<sourceFolder url="file://$MODULE_DIR$/src/test/resources" type="java-test-resource" /> |
|
| 15 |
<sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" /> |
|
| 16 |
<excludeFolder url="file://$MODULE_DIR$/target" /> |
|
| 17 |
</content> |
|
| 18 |
<orderEntry type="inheritedJdk" /> |
|
| 19 |
<orderEntry type="sourceFolder" forTests="false" /> |
|
| 20 |
<orderEntry type="library" name="Maven: eu.dnetlib:cnr-rmi-api:2.6.1" level="project" /> |
|
| 21 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-core:3.1.5" level="project" /> |
|
| 22 |
<orderEntry type="library" name="Maven: org.codehaus.woodstox:woodstox-core-asl:4.4.1" level="project" /> |
|
| 23 |
<orderEntry type="library" name="Maven: org.codehaus.woodstox:stax2-api:3.1.4" level="project" /> |
|
| 24 |
<orderEntry type="library" name="Maven: org.apache.ws.xmlschema:xmlschema-core:2.2.1" level="project" /> |
|
| 25 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-frontend-jaxws:3.1.5" level="project" /> |
|
| 26 |
<orderEntry type="library" name="Maven: xml-resolver:xml-resolver:1.2" level="project" /> |
|
| 27 |
<orderEntry type="library" name="Maven: org.ow2.asm:asm:5.0.4" level="project" /> |
|
| 28 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-bindings-soap:3.1.5" level="project" /> |
|
| 29 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-wsdl:3.1.5" level="project" /> |
|
| 30 |
<orderEntry type="library" name="Maven: wsdl4j:wsdl4j:1.6.3" level="project" /> |
|
| 31 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-databinding-jaxb:3.1.5" level="project" /> |
|
| 32 |
<orderEntry type="library" name="Maven: com.sun.xml.bind:jaxb-impl:2.2.11" level="project" /> |
|
| 33 |
<orderEntry type="library" name="Maven: com.sun.xml.bind:jaxb-core:2.2.11" level="project" /> |
|
| 34 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-bindings-xml:3.1.5" level="project" /> |
|
| 35 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-frontend-simple:3.1.5" level="project" /> |
|
| 36 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-ws-addr:3.1.5" level="project" /> |
|
| 37 |
<orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-ws-policy:3.1.5" level="project" /> |
|
| 38 |
<orderEntry type="library" name="Maven: org.apache.neethi:neethi:3.0.3" level="project" /> |
|
| 39 |
<orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" /> |
|
| 40 |
<orderEntry type="library" name="Maven: eu.dnetlib:cnr-service-common:2.1.6" level="project" /> |
|
| 41 |
<orderEntry type="library" name="Maven: org.quartz-scheduler:quartz:2.2.2" level="project" /> |
|
| 42 |
<orderEntry type="library" name="Maven: c3p0:c3p0:0.9.1.1" level="project" /> |
|
| 43 |
<orderEntry type="library" name="Maven: org.springframework:spring-tx:4.3.7.RELEASE" level="project" /> |
|
| 44 |
<orderEntry type="library" name="Maven: org.springframework:spring-beans:4.3.7.RELEASE" level="project" /> |
|
| 45 |
<orderEntry type="library" name="Maven: eu.dnetlib:cnr-misc-utils:1.0.4" level="project" /> |
|
| 46 |
<orderEntry type="library" name="Maven: commons-lang:commons-lang:2.6" level="project" /> |
|
| 47 |
<orderEntry type="library" name="Maven: jaxen:jaxen:1.1.6" level="project" /> |
|
| 48 |
<orderEntry type="library" name="Maven: saxonica:saxon:9.1.0.8" level="project" /> |
|
| 49 |
<orderEntry type="library" name="Maven: saxonica:saxon-dom:9.1.0.8" level="project" /> |
|
| 50 |
<orderEntry type="library" name="Maven: jgrapht:jgrapht:0.7.2" level="project" /> |
|
| 51 |
<orderEntry type="library" name="Maven: net.sf.ehcache:ehcache:2.10.3" level="project" /> |
|
| 52 |
<orderEntry type="library" name="Maven: eu.dnetlib:cnr-service-utils:1.0.2" level="project" /> |
|
| 53 |
<orderEntry type="library" name="Maven: apache:oro:2.0.8" level="project" /> |
|
| 54 |
<orderEntry type="library" name="Maven: org.springframework:spring-context:4.3.7.RELEASE" level="project" /> |
|
| 55 |
<orderEntry type="library" name="Maven: eu.dnetlib:cnr-spring-utils:1.0.1" level="project" /> |
|
| 56 |
<orderEntry type="library" name="Maven: org.springframework:spring-aop:4.3.7.RELEASE" level="project" /> |
|
| 57 |
<orderEntry type="library" name="Maven: jparsec:jparsec:2.0" level="project" /> |
|
| 58 |
<orderEntry type="library" name="Maven: runcc:runcc:0.7" level="project" /> |
|
| 59 |
<orderEntry type="module" module-name="dnet-openaireplus-mapping-utils" /> |
|
| 60 |
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.10" level="project" /> |
|
| 61 |
<orderEntry type="library" name="Maven: dom4j:dom4j:1.6.1" level="project" /> |
|
| 62 |
<orderEntry type="library" name="Maven: eu.dnetlib:dnet-openaire-data-protos:3.7.7" level="project" /> |
|
| 63 |
<orderEntry type="library" name="Maven: com.google.protobuf:protobuf-java:2.4.1" level="project" /> |
|
| 64 |
<orderEntry type="library" name="Maven: eu.dnetlib:dnet-pace-core:2.5.1" level="project" /> |
|
| 65 |
<orderEntry type="library" name="Maven: edu.cmu:secondstring:1.0.0" level="project" /> |
|
| 66 |
<orderEntry type="library" name="Maven: commons-collections:commons-collections:3.2.2" level="project" /> |
|
| 67 |
<orderEntry type="library" name="Maven: eu.dnetlib:dnet-index-solr-common:1.3.1" level="project" /> |
|
| 68 |
<orderEntry type="library" name="Maven: org.apache.solr:solr-solrj:5.5.4" level="project" /> |
|
| 69 |
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.5.3" level="project" /> |
|
| 70 |
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.4.6" level="project" /> |
|
| 71 |
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpmime:4.5.3" level="project" /> |
|
| 72 |
<orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.4.6" level="project" /> |
|
| 73 |
<orderEntry type="library" name="Maven: org.noggit:noggit:0.6" level="project" /> |
|
| 74 |
<orderEntry type="library" name="Maven: com.googlecode.protobuf-java-format:protobuf-java-format:1.2" level="project" /> |
|
| 75 |
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-web:1.5.2.RELEASE" level="project" /> |
|
| 76 |
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter:1.5.2.RELEASE" level="project" /> |
|
| 77 |
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot:1.5.2.RELEASE" level="project" /> |
|
| 78 |
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-autoconfigure:1.5.2.RELEASE" level="project" /> |
|
| 79 |
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-logging:1.5.2.RELEASE" level="project" /> |
|
| 80 |
<orderEntry type="library" name="Maven: ch.qos.logback:logback-classic:1.1.11" level="project" /> |
|
| 81 |
<orderEntry type="library" name="Maven: ch.qos.logback:logback-core:1.1.11" level="project" /> |
|
| 82 |
<orderEntry type="library" name="Maven: org.slf4j:jul-to-slf4j:1.7.24" level="project" /> |
|
| 83 |
<orderEntry type="library" name="Maven: org.slf4j:log4j-over-slf4j:1.7.24" level="project" /> |
|
| 84 |
<orderEntry type="library" scope="RUNTIME" name="Maven: org.yaml:snakeyaml:1.17" level="project" /> |
|
| 85 |
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-tomcat:1.5.2.RELEASE" level="project" /> |
|
| 86 |
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-core:8.5.11" level="project" /> |
|
| 87 |
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-el:8.5.11" level="project" /> |
|
| 88 |
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-websocket:8.5.11" level="project" /> |
|
| 89 |
<orderEntry type="library" name="Maven: org.hibernate:hibernate-validator:5.3.4.Final" level="project" /> |
|
| 90 |
<orderEntry type="library" name="Maven: javax.validation:validation-api:1.1.0.Final" level="project" /> |
|
| 91 |
<orderEntry type="library" name="Maven: org.jboss.logging:jboss-logging:3.3.0.Final" level="project" /> |
|
| 92 |
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.8.7" level="project" /> |
|
| 93 |
<orderEntry type="library" name="Maven: org.springframework:spring-web:4.3.7.RELEASE" level="project" /> |
|
| 94 |
<orderEntry type="library" name="Maven: org.springframework:spring-webmvc:4.3.7.RELEASE" level="project" /> |
|
| 95 |
<orderEntry type="library" name="Maven: org.springframework:spring-expression:4.3.7.RELEASE" level="project" /> |
|
| 96 |
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-jdbc:1.5.2.RELEASE" level="project" /> |
|
| 97 |
<orderEntry type="library" name="Maven: org.apache.tomcat:tomcat-jdbc:8.5.11" level="project" /> |
|
| 98 |
<orderEntry type="library" name="Maven: org.apache.tomcat:tomcat-juli:8.5.11" level="project" /> |
|
| 99 |
<orderEntry type="library" name="Maven: org.springframework:spring-jdbc:4.3.7.RELEASE" level="project" /> |
|
| 100 |
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-data-jpa:1.5.2.RELEASE" level="project" /> |
|
| 101 |
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-aop:1.5.2.RELEASE" level="project" /> |
|
| 102 |
<orderEntry type="library" name="Maven: org.aspectj:aspectjweaver:1.8.9" level="project" /> |
|
| 103 |
<orderEntry type="library" name="Maven: org.hibernate:hibernate-core:5.0.12.Final" level="project" /> |
|
| 104 |
<orderEntry type="library" name="Maven: org.hibernate.javax.persistence:hibernate-jpa-2.1-api:1.0.0.Final" level="project" /> |
|
| 105 |
<orderEntry type="library" name="Maven: org.javassist:javassist:3.21.0-GA" level="project" /> |
|
| 106 |
<orderEntry type="library" name="Maven: org.jboss:jandex:2.0.0.Final" level="project" /> |
|
| 107 |
<orderEntry type="library" name="Maven: org.hibernate.common:hibernate-commons-annotations:5.0.1.Final" level="project" /> |
|
| 108 |
<orderEntry type="library" name="Maven: org.hibernate:hibernate-entitymanager:5.0.12.Final" level="project" /> |
|
| 109 |
<orderEntry type="library" name="Maven: javax.transaction:javax.transaction-api:1.2" level="project" /> |
|
| 110 |
<orderEntry type="library" name="Maven: org.springframework.data:spring-data-jpa:1.11.1.RELEASE" level="project" /> |
|
| 111 |
<orderEntry type="library" name="Maven: org.springframework.data:spring-data-commons:1.13.1.RELEASE" level="project" /> |
|
| 112 |
<orderEntry type="library" name="Maven: org.springframework:spring-orm:4.3.7.RELEASE" level="project" /> |
|
| 113 |
<orderEntry type="library" name="Maven: org.slf4j:jcl-over-slf4j:1.7.24" level="project" /> |
|
| 114 |
<orderEntry type="library" name="Maven: org.springframework:spring-aspects:4.3.7.RELEASE" level="project" /> |
|
| 115 |
<orderEntry type="library" name="Maven: org.postgresql:postgresql:42.0.0" level="project" /> |
|
| 116 |
<orderEntry type="library" name="Maven: org.apache.commons:commons-dbcp2:2.1" level="project" /> |
|
| 117 |
<orderEntry type="library" name="Maven: org.apache.commons:commons-pool2:2.4.2" level="project" /> |
|
| 118 |
<orderEntry type="library" name="Maven: commons-logging:commons-logging:1.2" level="project" /> |
|
| 119 |
<orderEntry type="library" name="Maven: org.antlr:stringtemplate:3.2.1" level="project" /> |
|
| 120 |
<orderEntry type="library" name="Maven: antlr:antlr:2.7.7" level="project" /> |
|
| 121 |
<orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.5" level="project" /> |
|
| 122 |
<orderEntry type="library" name="Maven: commons-io:commons-io:2.5" level="project" /> |
|
| 123 |
<orderEntry type="library" name="Maven: com.google.guava:guava:21.0" level="project" /> |
|
| 124 |
<orderEntry type="library" name="Maven: net.sf.supercsv:super-csv:2.4.0" level="project" /> |
|
| 125 |
<orderEntry type="library" name="Maven: com.google.code.gson:gson:2.8.0" level="project" /> |
|
| 126 |
<orderEntry type="library" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-joda:2.6.6" level="project" /> |
|
| 127 |
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.8.0" level="project" /> |
|
| 128 |
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.8.7" level="project" /> |
|
| 129 |
<orderEntry type="library" name="Maven: joda-time:joda-time:2.8.2" level="project" /> |
|
| 130 |
<orderEntry type="library" name="Maven: org.mongodb:mongo-java-driver:3.4.2" level="project" /> |
|
| 131 |
<orderEntry type="library" name="Maven: io.springfox:springfox-swagger2:2.7.0" level="project" /> |
|
| 132 |
<orderEntry type="library" name="Maven: io.swagger:swagger-annotations:1.5.13" level="project" /> |
|
| 133 |
<orderEntry type="library" name="Maven: io.swagger:swagger-models:1.5.13" level="project" /> |
|
| 134 |
<orderEntry type="library" name="Maven: io.springfox:springfox-spi:2.7.0" level="project" /> |
|
| 135 |
<orderEntry type="library" name="Maven: io.springfox:springfox-core:2.7.0" level="project" /> |
|
| 136 |
<orderEntry type="library" name="Maven: net.bytebuddy:byte-buddy:1.6.14" level="project" /> |
|
| 137 |
<orderEntry type="library" name="Maven: io.springfox:springfox-schema:2.7.0" level="project" /> |
|
| 138 |
<orderEntry type="library" name="Maven: io.springfox:springfox-swagger-common:2.7.0" level="project" /> |
|
| 139 |
<orderEntry type="library" name="Maven: io.springfox:springfox-spring-web:2.7.0" level="project" /> |
|
| 140 |
<orderEntry type="library" name="Maven: org.reflections:reflections:0.9.11" level="project" /> |
|
| 141 |
<orderEntry type="library" name="Maven: com.fasterxml:classmate:1.3.3" level="project" /> |
|
| 142 |
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.24" level="project" /> |
|
| 143 |
<orderEntry type="library" name="Maven: org.springframework.plugin:spring-plugin-core:1.2.0.RELEASE" level="project" /> |
|
| 144 |
<orderEntry type="library" name="Maven: org.springframework.plugin:spring-plugin-metadata:1.2.0.RELEASE" level="project" /> |
|
| 145 |
<orderEntry type="library" name="Maven: org.mapstruct:mapstruct:1.1.0.Final" level="project" /> |
|
| 146 |
<orderEntry type="library" name="Maven: io.springfox:springfox-swagger-ui:2.7.0" level="project" /> |
|
| 147 |
<orderEntry type="library" scope="TEST" name="Maven: org.springframework.boot:spring-boot-starter-test:1.5.2.RELEASE" level="project" /> |
|
| 148 |
<orderEntry type="library" scope="TEST" name="Maven: org.springframework.boot:spring-boot-test:1.5.2.RELEASE" level="project" /> |
|
| 149 |
<orderEntry type="library" scope="TEST" name="Maven: org.springframework.boot:spring-boot-test-autoconfigure:1.5.2.RELEASE" level="project" /> |
|
| 150 |
<orderEntry type="library" scope="TEST" name="Maven: com.jayway.jsonpath:json-path:2.2.0" level="project" /> |
|
| 151 |
<orderEntry type="library" scope="TEST" name="Maven: net.minidev:json-smart:2.2.1" level="project" /> |
|
| 152 |
<orderEntry type="library" scope="TEST" name="Maven: net.minidev:accessors-smart:1.1" level="project" /> |
|
| 153 |
<orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.12" level="project" /> |
|
| 154 |
<orderEntry type="library" scope="TEST" name="Maven: org.assertj:assertj-core:2.6.0" level="project" /> |
|
| 155 |
<orderEntry type="library" scope="TEST" name="Maven: org.mockito:mockito-core:1.10.19" level="project" /> |
|
| 156 |
<orderEntry type="library" scope="TEST" name="Maven: org.objenesis:objenesis:2.1" level="project" /> |
|
| 157 |
<orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" /> |
|
| 158 |
<orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-library:1.3" level="project" /> |
|
| 159 |
<orderEntry type="library" scope="TEST" name="Maven: org.skyscreamer:jsonassert:1.4.0" level="project" /> |
|
| 160 |
<orderEntry type="library" scope="TEST" name="Maven: com.vaadin.external.google:android-json:0.0.20131108.vaadin1" level="project" /> |
|
| 161 |
<orderEntry type="library" name="Maven: org.springframework:spring-core:4.3.7.RELEASE" level="project" /> |
|
| 162 |
<orderEntry type="library" name="Maven: org.springframework:spring-test:4.3.7.RELEASE" level="project" /> |
|
| 163 |
</component> |
|
| 164 |
</module> |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/deploy.info | ||
|---|---|---|
| 1 |
{"type_source": "SVN", "goal": "package -U source:jar",
|
|
| 2 |
"url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-openaire-projects-exporter/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", |
|
| 3 |
"deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-openaire-projects-exporter"} |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/test/java/eu/dnetlib/openaire/exporter/model/project/ProjectDetailsTest.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.openaire.exporter.model.project; |
|
| 2 |
|
|
| 3 |
import java.io.IOException; |
|
| 4 |
|
|
| 5 |
import org.junit.Test; |
|
| 6 |
|
|
| 7 |
/** |
|
| 8 |
* Created by claudio on 05/07/2017. |
|
| 9 |
*/ |
|
| 10 |
public class ProjectDetailsTest {
|
|
| 11 |
|
|
| 12 |
@Test |
|
| 13 |
public void testCSV() throws IOException {
|
|
| 14 |
|
|
| 15 |
final ProjectDetails p = ProjectDetails.fromCSV( |
|
| 16 |
"arc_________::ANZCCART,,ANZCCART,{},\"[\"\"\\u003cfundingtree\\u003e\\n \\u003cfunder\\u003e\\n \\u003cid\\u003earc_________::ARC\\u003c/id\\u003e\\n \\u003cshortname\\u003eARC\\u003c/shortname\\u003e\\n \\u003cname\\u003eAustralian Research Council (ARC)\\u003c/name\\u003e\\n \\u003cjurisdiction\\u003eAU\\u003c/jurisdiction\\u003e\\n \\u003c/funder\\u003e\\n \\u003cfunding_level_0\\u003e\\n \\u003cid\\u003earc_________::ARC::Special Research initiative (Australian and New Zealand Council for the Care of Animals in Research and Teaching)\\u003c/id\\u003e\\n \\u003cname\\u003eSpecial Research initiative (Australian and New Zealand Council for the Care of Animals in Research and Teaching)\\u003c/name\\u003e\\n \\u003cdescription\\u003eSpecial Research initiative (Australian and New Zealand Council for the Care of Animals in Research and Teaching)\\u003c/description\\u003e\\n \\u003cparent/\\u003e\\n \\u003cclass\\u003earc:fundingStream\\u003c/class\\u003e\\n \\u003c/funding_level_0\\u003e\\n \\u003c/fundingtree\\u003e\"\"]\"");
|
|
| 17 |
|
|
| 18 |
System.out.println(p.asJson()); |
|
| 19 |
|
|
| 20 |
System.out.println(p.asCSV()); |
|
| 21 |
|
|
| 22 |
|
|
| 23 |
} |
|
| 24 |
|
|
| 25 |
} |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/test/java/eu/dnetlib/openaire/exporter/project/ProjectsControllerTest.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.openaire.exporter.project; |
|
| 2 |
|
|
| 3 |
import java.io.IOException; |
|
| 4 |
|
|
| 5 |
import org.antlr.stringtemplate.StringTemplate; |
|
| 6 |
import org.apache.commons.io.IOUtils; |
|
| 7 |
import org.apache.commons.logging.Log; |
|
| 8 |
import org.apache.commons.logging.LogFactory; |
|
| 9 |
import org.junit.Before; |
|
| 10 |
import org.junit.Ignore; |
|
| 11 |
import org.junit.Test; |
|
| 12 |
import org.junit.runner.RunWith; |
|
| 13 |
import org.springframework.boot.test.context.SpringBootTest; |
|
| 14 |
import org.springframework.core.io.ClassPathResource; |
|
| 15 |
import org.springframework.core.io.Resource; |
|
| 16 |
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; |
|
| 17 |
|
|
| 18 |
import static org.junit.Assert.assertEquals; |
|
| 19 |
|
|
| 20 |
@Ignore |
|
| 21 |
@SpringBootTest |
|
| 22 |
@RunWith(SpringJUnit4ClassRunner.class) |
|
| 23 |
public class ProjectsControllerTest {
|
|
| 24 |
|
|
| 25 |
private static final Log log = LogFactory.getLog(ProjectsControllerTest.class); |
|
| 26 |
private String queryTemplate = "/eu/dnetlib/openaire/exporter/sql/projects_fundings.sql.st"; |
|
| 27 |
|
|
| 28 |
private Resource expectedQueryTemplate = new ClassPathResource("/eu/dnetlib/openaire/exporter/sql/expected_projects_fundings.sql.st");
|
|
| 29 |
|
|
| 30 |
private ProjectsController controller; |
|
| 31 |
private ProjectQueryParams params; |
|
| 32 |
|
|
| 33 |
@Before |
|
| 34 |
public void setup() {
|
|
| 35 |
controller = new ProjectsController(); |
|
| 36 |
Resource template = new ClassPathResource(queryTemplate); |
|
| 37 |
|
|
| 38 |
//TODO reimplement bean injection for testing |
|
| 39 |
//controller.setProjectsFundingQueryTemplate(template); |
|
| 40 |
params = new ProjectQueryParams(); |
|
| 41 |
} |
|
| 42 |
|
|
| 43 |
@Test |
|
| 44 |
public void testObtainFP7Query() throws IllegalArgumentException, IOException {
|
|
| 45 |
params.setFundingProgramme("FP7");
|
|
| 46 |
params.setFundingPath(null); |
|
| 47 |
String res = controller.obtainQuery(params); |
|
| 48 |
final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); |
|
| 49 |
st.setAttribute("fundingprefix", "ec__________::EC::FP7");
|
|
| 50 |
log.debug(res); |
|
| 51 |
log.debug(st); |
|
| 52 |
assertEquals(st.toString(), res); |
|
| 53 |
} |
|
| 54 |
|
|
| 55 |
@Test |
|
| 56 |
public void testObtainFP7QuerySP1() throws IllegalArgumentException, IOException {
|
|
| 57 |
params.setFundingProgramme("FP7");
|
|
| 58 |
params.setFundingPath("SP1");
|
|
| 59 |
String res = controller.obtainQuery(params); |
|
| 60 |
final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); |
|
| 61 |
st.setAttribute("fundingprefix", "ec__________::EC::FP7::SP1");
|
|
| 62 |
log.debug(res); |
|
| 63 |
assertEquals(st.toString(), res); |
|
| 64 |
} |
|
| 65 |
|
|
| 66 |
@Test |
|
| 67 |
public void testObtainFP7QueryHealth() throws IllegalArgumentException, IOException {
|
|
| 68 |
params.setFundingProgramme("FP7");
|
|
| 69 |
params.setFundingPath("SP1::HEALTH");
|
|
| 70 |
String res = controller.obtainQuery(params); |
|
| 71 |
final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); |
|
| 72 |
st.setAttribute("fundingprefix", "ec__________::EC::FP7::SP1::HEALTH");
|
|
| 73 |
log.debug(res); |
|
| 74 |
assertEquals(st.toString(), res); |
|
| 75 |
} |
|
| 76 |
|
|
| 77 |
@Test |
|
| 78 |
public void testObtainFP7QueryHealth2() throws IllegalArgumentException, IOException {
|
|
| 79 |
params.setFundingProgramme("FP7");
|
|
| 80 |
params.setFundingPath("%::HEALTH");
|
|
| 81 |
String res = controller.obtainQuery(params); |
|
| 82 |
final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); |
|
| 83 |
st.setAttribute("fundingprefix", "ec__________::EC::FP7::%::HEALTH");
|
|
| 84 |
log.debug(res); |
|
| 85 |
assertEquals(st.toString(), res); |
|
| 86 |
} |
|
| 87 |
|
|
| 88 |
@Test |
|
| 89 |
public void testObtainWellcomeTrustQuery() throws IllegalArgumentException, IOException {
|
|
| 90 |
params.setFundingProgramme("WT");
|
|
| 91 |
params.setFundingPath(null); |
|
| 92 |
String res = controller.obtainQuery(params); |
|
| 93 |
final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); |
|
| 94 |
st.setAttribute("fundingprefix", "wt__________::WT");
|
|
| 95 |
log.debug(res); |
|
| 96 |
assertEquals(st.toString(), res); |
|
| 97 |
} |
|
| 98 |
|
|
| 99 |
@Test |
|
| 100 |
public void testObtainFCTQuery() throws IllegalArgumentException, IOException {
|
|
| 101 |
params.setFundingProgramme("FCT");
|
|
| 102 |
params.setFundingPath(null); |
|
| 103 |
String res = controller.obtainQuery(params); |
|
| 104 |
final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); |
|
| 105 |
st.setAttribute("fundingprefix", "fct_________::FCT");
|
|
| 106 |
log.debug(res); |
|
| 107 |
assertEquals(st.toString(), res); |
|
| 108 |
} |
|
| 109 |
|
|
| 110 |
@Test |
|
| 111 |
public void testQueryWithDateParams() throws IllegalArgumentException, IOException {
|
|
| 112 |
params.setFundingProgramme("WT");
|
|
| 113 |
params.setFundingPath(null); |
|
| 114 |
params.setStartFrom("2015-05-04");
|
|
| 115 |
String res = controller.obtainQuery(params); |
|
| 116 |
log.debug(res); |
|
| 117 |
final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); |
|
| 118 |
st.setAttribute("fundingprefix", "wt__________::WT");
|
|
| 119 |
String q = st.toString() + " AND startdate >= '2015-05-04'"; |
|
| 120 |
assertEquals(q, res); |
|
| 121 |
} |
|
| 122 |
|
|
| 123 |
@Test |
|
| 124 |
public void testObtainSNSFQuery() throws IllegalArgumentException, IOException {
|
|
| 125 |
params.setFundingProgramme("SNSF");
|
|
| 126 |
params.setFundingPath(null); |
|
| 127 |
String res = controller.obtainQuery(params); |
|
| 128 |
final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); |
|
| 129 |
st.setAttribute("fundingprefix", "snsf________::SNSF");
|
|
| 130 |
log.debug(res); |
|
| 131 |
assertEquals(st.toString(), res); |
|
| 132 |
} |
|
| 133 |
|
|
| 134 |
} |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/test/java/eu/dnetlib/openaire/exporter/project/ProjectQueryParamsTest.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.openaire.exporter.project; |
|
| 2 |
|
|
| 3 |
import eu.dnetlib.openaire.exporter.project.ProjectQueryParams; |
|
| 4 |
import org.junit.Before; |
|
| 5 |
import org.junit.Test; |
|
| 6 |
|
|
| 7 |
import static org.junit.Assert.assertEquals; |
|
| 8 |
|
|
| 9 |
public class ProjectQueryParamsTest {
|
|
| 10 |
|
|
| 11 |
private ProjectQueryParams queryParams; |
|
| 12 |
|
|
| 13 |
@Before |
|
| 14 |
public void setUp() throws Exception {
|
|
| 15 |
queryParams = new ProjectQueryParams(); |
|
| 16 |
|
|
| 17 |
} |
|
| 18 |
|
|
| 19 |
@Test |
|
| 20 |
public void testVerifyParamWhiteSpace() {
|
|
| 21 |
queryParams.verifyParam("Discovery Projects");
|
|
| 22 |
} |
|
| 23 |
|
|
| 24 |
@Test |
|
| 25 |
public void testVerifyParamPercentage() {
|
|
| 26 |
queryParams.verifyParam("Discovery%20Projects");
|
|
| 27 |
} |
|
| 28 |
|
|
| 29 |
@Test |
|
| 30 |
public void testVerifyDateParam(){
|
|
| 31 |
String correctDate = "2012-03-04"; |
|
| 32 |
assertEquals(correctDate, queryParams.verifyDateParam(correctDate)); |
|
| 33 |
|
|
| 34 |
} |
|
| 35 |
|
|
| 36 |
@Test(expected=IllegalArgumentException.class) |
|
| 37 |
public void testVerifyDateParamException(){
|
|
| 38 |
String wrongDate = "12-12-12"; |
|
| 39 |
queryParams.verifyDateParam(wrongDate); |
|
| 40 |
|
|
| 41 |
} |
|
| 42 |
} |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/test/resources/eu/dnetlib/openaire/exporter/MSES-project.xml | ||
|---|---|---|
| 1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
| 2 |
<ROW> |
|
| 3 |
<FIELD name="funder">MSES</FIELD> |
|
| 4 |
<FIELD name="end_date">2009-12-31</FIELD> |
|
| 5 |
<FIELD name="grant_agreement_number">001-0000000-3177</FIELD> |
|
| 6 |
<FIELD name="title">Residence time of bacteria Escherichia coli in seawater and marine organisms</FIELD> |
|
| 7 |
<FIELD name="acronym" isNull="true"/> |
|
| 8 |
<FIELD name="fundingpathid">irb_hr______::MSES::fundingStream</FIELD> |
|
| 9 |
<FIELD name="jurisdiction">HR</FIELD> |
|
| 10 |
<FIELD name="start_date">2007-01-01</FIELD> |
|
| 11 |
</ROW> |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/test/resources/eu/dnetlib/openaire/exporter/WT-project.xml | ||
|---|---|---|
| 1 |
<ROW> |
|
| 2 |
<FIELD name="funder">WT</FIELD> |
|
| 3 |
<FIELD name="end_date">2013-07-29</FIELD> |
|
| 4 |
<FIELD name="grant_agreement_number">098241</FIELD> |
|
| 5 |
<FIELD name="title">EYEDIO DIGI - DEVELOPMENT OF AN INNOVATIVE, AFFORDABLE, EASY TO USE, HANDHELD RETINAL IMAGING PRODUCT FOR THE DIAGNOSIS OF DIABETIC RETINOPATHY.</FIELD> |
|
| 6 |
<FIELD name="acronym"></FIELD> |
|
| 7 |
<FIELD name="fundingpathid">wt__________::WT::Immunology and Infectious Disease</FIELD> |
|
| 8 |
<FIELD name="start_date">2012-01-30</FIELD> |
|
| 9 |
<FIELD name="jurisdiction"/> |
|
| 10 |
<FIELD name="description">description of the funding path</FIELD> |
|
| 11 |
</ROW> |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/test/resources/eu/dnetlib/openaire/exporter/FCT-project.xml | ||
|---|---|---|
| 1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
| 2 |
<ROW> |
|
| 3 |
<FIELD name="funder">FCT</FIELD> |
|
| 4 |
<FIELD name="jurisdiction">PT</FIELD> |
|
| 5 |
<FIELD name="end_date">2003-10-31</FIELD> |
|
| 6 |
<FIELD name="grant_agreement_number">32639</FIELD> |
|
| 7 |
<FIELD name="title">Social Classes and «lifestyles» in the city of Oporto</FIELD> |
|
| 8 |
<FIELD name="fundingpathid">fct_________::FCT::Orçamento de Funcionamento/POSC</FIELD> |
|
| 9 |
<FIELD name="acronym">PTDC/FIL-FIL/109889/2009</FIELD> |
|
| 10 |
<FIELD name="start_date">2000-11-01</FIELD> |
|
| 11 |
<FIELD name="description">description of the funding path</FIELD> |
|
| 12 |
</ROW> |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/test/resources/eu/dnetlib/openaire/exporter/FP7-project.xml | ||
|---|---|---|
| 1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
| 2 |
<ROW> |
|
| 3 |
<FIELD name="funder">EC</FIELD> |
|
| 4 |
<FIELD name="jurisdiction">EU</FIELD> |
|
| 5 |
<FIELD name="end_date">2012-07-31</FIELD> |
|
| 6 |
<FIELD name="grant_agreement_number">255646</FIELD> |
|
| 7 |
<FIELD name="title">Semiconductor lasers for generation of non-diffracting (Bessel) beams.</FIELD> |
|
| 8 |
<FIELD name="fundingpathid">ec__________::EC::FP7::SP3::PEOPLE</FIELD> |
|
| 9 |
<FIELD name="acronym">SENDBEAMS</FIELD> |
|
| 10 |
<FIELD name="start_date">2010-08-01</FIELD> |
|
| 11 |
<FIELD name="description">description of the funding path</FIELD> |
|
| 12 |
</ROW> |
|
| 13 |
|
|
| 14 |
|
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/test/resources/eu/dnetlib/openaire/exporter/sql/expected_projects_fundings.sql.st | ||
|---|---|---|
| 1 |
SELECT |
|
| 2 |
funder, |
|
| 3 |
jurisdiction, |
|
| 4 |
fundingpathid, |
|
| 5 |
acronym, |
|
| 6 |
title, |
|
| 7 |
code, |
|
| 8 |
startdate, |
|
| 9 |
enddate |
|
| 10 |
FROM projects_api |
|
| 11 |
WHERE fundingpathid like '$fundingprefix$%' |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/test/resources/eu/dnetlib/openaire/exporter/SFI-project.xml | ||
|---|---|---|
| 1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
| 2 |
<ROW> |
|
| 3 |
<FIELD name="funder">SFI</FIELD> |
|
| 4 |
<FIELD name="end_date">2016-08-31</FIELD> |
|
| 5 |
<FIELD name="grant_agreement_number">14/ADV/RC3022</FIELD> |
|
| 6 |
<FIELD name="title">Dr. Vesna Jaksic</FIELD> |
|
| 7 |
<FIELD name="acronym" isNull="true"/> |
|
| 8 |
<FIELD name="fundingpathid">sfi_________::SFI::Advance Award Programme</FIELD> |
|
| 9 |
<FIELD name="jurisdiction" isNull="true"/> |
|
| 10 |
<FIELD name="start_date">2014-09-01</FIELD> |
|
| 11 |
</ROW> |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/test/resources/logback-test.xml | ||
|---|---|---|
| 1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
| 2 |
<configuration> |
|
| 3 |
<include resource="org/springframework/boot/logging/logback/base.xml"/> |
|
| 4 |
<logger name="org.springframework" level="OFF"> |
|
| 5 |
<appender-ref ref="CONSOLE" /> |
|
| 6 |
</logger> |
|
| 7 |
<logger name="root" level="OFF"> |
|
| 8 |
<appender-ref ref="CONSOLE" /> |
|
| 9 |
</logger> |
|
| 10 |
<logger name="eu.dnetlib" level="INFO"> |
|
| 11 |
<appender-ref ref="CONSOLE" /> |
|
| 12 |
</logger> |
|
| 13 |
<logger name="eu.dnetlib.openaire.exporter" level="INFO"> |
|
| 14 |
<appender-ref ref="CONSOLE" /> |
|
| 15 |
</logger> |
|
| 16 |
</configuration> |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/main/java/eu/dnetlib/swagger/configuration/SwaggerDocumentationConfig.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.swagger.configuration; |
|
| 2 |
|
|
| 3 |
import eu.dnetlib.OpenaireExporterConfig; |
|
| 4 |
import eu.dnetlib.OpenaireExporterConfig.Swagger; |
|
| 5 |
import eu.dnetlib.common.rmi.DNetRestDocumentation; |
|
| 6 |
import org.springframework.beans.factory.annotation.Autowired; |
|
| 7 |
import org.springframework.beans.factory.annotation.Value; |
|
| 8 |
import org.springframework.context.annotation.Bean; |
|
| 9 |
import org.springframework.context.annotation.Configuration; |
|
| 10 |
import springfox.documentation.builders.ApiInfoBuilder; |
|
| 11 |
import springfox.documentation.builders.RequestHandlerSelectors; |
|
| 12 |
import springfox.documentation.service.ApiInfo; |
|
| 13 |
import springfox.documentation.service.Contact; |
|
| 14 |
import springfox.documentation.spi.DocumentationType; |
|
| 15 |
import springfox.documentation.spring.web.plugins.Docket; |
|
| 16 |
import springfox.documentation.swagger2.annotations.EnableSwagger2; |
|
| 17 |
|
|
| 18 |
@Configuration |
|
| 19 |
@EnableSwagger2 |
|
| 20 |
public class SwaggerDocumentationConfig {
|
|
| 21 |
|
|
| 22 |
@Autowired |
|
| 23 |
private OpenaireExporterConfig config; |
|
| 24 |
|
|
| 25 |
@Bean |
|
| 26 |
public Docket customImplementation() {
|
|
| 27 |
return new Docket(DocumentationType.SWAGGER_2) |
|
| 28 |
.select() |
|
| 29 |
.apis(RequestHandlerSelectors.withClassAnnotation(DNetRestDocumentation.class)) |
|
| 30 |
.build() |
|
| 31 |
.directModelSubstitute(org.joda.time.LocalDate.class, java.sql.Date.class) |
|
| 32 |
.directModelSubstitute(org.joda.time.DateTime.class, java.util.Date.class) |
|
| 33 |
.apiInfo(apiInfo()); |
|
| 34 |
} |
|
| 35 |
|
|
| 36 |
private ApiInfo apiInfo() {
|
|
| 37 |
final Swagger swag = config.getSwagger(); |
|
| 38 |
return new ApiInfoBuilder() |
|
| 39 |
.title(swag.getApiTitle()) |
|
| 40 |
.description(swag.getApiDescription()) |
|
| 41 |
.license(swag.getApiLicense()) |
|
| 42 |
.licenseUrl(swag.getApiLicenseUrl()) |
|
| 43 |
.termsOfServiceUrl("")
|
|
| 44 |
.version("1.0.0")
|
|
| 45 |
.contact(new Contact( |
|
| 46 |
swag.getApiContactName(), |
|
| 47 |
swag.getApiContactUrl(), |
|
| 48 |
swag.getApiContactEmail())) |
|
| 49 |
.build(); |
|
| 50 |
} |
|
| 51 |
|
|
| 52 |
} |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/main/java/eu/dnetlib/swagger/configuration/RFC3339DateFormat.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.swagger.configuration; |
|
| 2 |
|
|
| 3 |
import java.text.FieldPosition; |
|
| 4 |
import java.util.Date; |
|
| 5 |
|
|
| 6 |
import com.fasterxml.jackson.databind.util.ISO8601DateFormat; |
|
| 7 |
import com.fasterxml.jackson.databind.util.ISO8601Utils; |
|
| 8 |
|
|
| 9 |
public class RFC3339DateFormat extends ISO8601DateFormat {
|
|
| 10 |
|
|
| 11 |
// Same as ISO8601DateFormat but serializing milliseconds. |
|
| 12 |
@Override |
|
| 13 |
public StringBuffer format(Date date, StringBuffer toAppendTo, FieldPosition fieldPosition) {
|
|
| 14 |
String value = ISO8601Utils.format(date, true); |
|
| 15 |
toAppendTo.append(value); |
|
| 16 |
return toAppendTo; |
|
| 17 |
} |
|
| 18 |
|
|
| 19 |
} |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/main/java/eu/dnetlib/Utils.java | ||
|---|---|---|
| 1 |
package eu.dnetlib; |
|
| 2 |
|
|
| 3 |
import java.util.Iterator; |
|
| 4 |
import java.util.Spliterator; |
|
| 5 |
import java.util.Spliterators; |
|
| 6 |
import java.util.stream.Stream; |
|
| 7 |
import java.util.stream.StreamSupport; |
|
| 8 |
|
|
| 9 |
/** |
|
| 10 |
* Created by claudio on 13/04/2017. |
|
| 11 |
*/ |
|
| 12 |
public class Utils {
|
|
| 13 |
|
|
| 14 |
public static <T> Stream<T> stream(Iterator<T> iterator) {
|
|
| 15 |
return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false); |
|
| 16 |
} |
|
| 17 |
|
|
| 18 |
} |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/main/java/eu/dnetlib/OpenaireExporterConfig.java | ||
|---|---|---|
| 1 |
package eu.dnetlib; |
|
| 2 |
|
|
| 3 |
import org.springframework.beans.factory.annotation.Value; |
|
| 4 |
import org.springframework.boot.context.properties.ConfigurationProperties; |
|
| 5 |
import org.springframework.core.io.ClassPathResource; |
|
| 6 |
import org.springframework.core.io.Resource; |
|
| 7 |
import org.springframework.stereotype.Component; |
|
| 8 |
|
|
| 9 |
/** |
|
| 10 |
* Created by Alessia Bardi on 31/03/17. |
|
| 11 |
* |
|
| 12 |
* @author Alessia Bardi |
|
| 13 |
*/ |
|
| 14 |
@Component |
|
| 15 |
@ConfigurationProperties(prefix = "openaire.exporter") |
|
| 16 |
public class OpenaireExporterConfig {
|
|
| 17 |
|
|
| 18 |
// ISLOOKUP |
|
| 19 |
private ClassPathResource findSolrIndexUrl; |
|
| 20 |
private ClassPathResource findIndexDsInfo; |
|
| 21 |
private String isLookupUrl; |
|
| 22 |
private int requestWorkers = 100; |
|
| 23 |
private int requestTimeout = 10; |
|
| 24 |
|
|
| 25 |
private Datasource datasource; |
|
| 26 |
private Project project; |
|
| 27 |
private Jdbc jdbc; |
|
| 28 |
private Swagger swagger; |
|
| 29 |
|
|
| 30 |
public static class Datasource {
|
|
| 31 |
// MONGODB |
|
| 32 |
private String mongoHost; |
|
| 33 |
private int mongoPort; |
|
| 34 |
private String mongoCollectionName; |
|
| 35 |
private String mongoDbName; |
|
| 36 |
private int mongoConnectionsPerHost; |
|
| 37 |
private int mongoQueryLimit; |
|
| 38 |
|
|
| 39 |
public String getMongoHost() {
|
|
| 40 |
return mongoHost; |
|
| 41 |
} |
|
| 42 |
|
|
| 43 |
public void setMongoHost(final String mongoHost) {
|
|
| 44 |
this.mongoHost = mongoHost; |
|
| 45 |
} |
|
| 46 |
|
|
| 47 |
public int getMongoPort() {
|
|
| 48 |
return mongoPort; |
|
| 49 |
} |
|
| 50 |
|
|
| 51 |
public void setMongoPort(final int mongoPort) {
|
|
| 52 |
this.mongoPort = mongoPort; |
|
| 53 |
} |
|
| 54 |
|
|
| 55 |
public String getMongoCollectionName() {
|
|
| 56 |
return mongoCollectionName; |
|
| 57 |
} |
|
| 58 |
|
|
| 59 |
public void setMongoCollectionName(final String mongoCollectionName) {
|
|
| 60 |
this.mongoCollectionName = mongoCollectionName; |
|
| 61 |
} |
|
| 62 |
|
|
| 63 |
public String getMongoDbName() {
|
|
| 64 |
return mongoDbName; |
|
| 65 |
} |
|
| 66 |
|
|
| 67 |
public void setMongoDbName(final String mongoDbName) {
|
|
| 68 |
this.mongoDbName = mongoDbName; |
|
| 69 |
} |
|
| 70 |
|
|
| 71 |
public int getMongoConnectionsPerHost() {
|
|
| 72 |
return mongoConnectionsPerHost; |
|
| 73 |
} |
|
| 74 |
|
|
| 75 |
public void setMongoConnectionsPerHost(final int mongoConnectionsPerHost) {
|
|
| 76 |
this.mongoConnectionsPerHost = mongoConnectionsPerHost; |
|
| 77 |
} |
|
| 78 |
|
|
| 79 |
public int getMongoQueryLimit() {
|
|
| 80 |
return mongoQueryLimit; |
|
| 81 |
} |
|
| 82 |
|
|
| 83 |
public void setMongoQueryLimit(final int mongoQueryLimit) {
|
|
| 84 |
this.mongoQueryLimit = mongoQueryLimit; |
|
| 85 |
} |
|
| 86 |
} |
|
| 87 |
|
|
| 88 |
public static class Project {
|
|
| 89 |
|
|
| 90 |
private int gzipFlushSize; |
|
| 91 |
private String tsvFields; |
|
| 92 |
private Resource projectsFundingQueryTemplate; |
|
| 93 |
private Resource dspaceTemplate; |
|
| 94 |
private Resource dspaceHeadTemplate; |
|
| 95 |
private Resource dspaceTailTemplate; |
|
| 96 |
private Resource eprintsTemplate; |
|
| 97 |
|
|
| 98 |
public int getGzipFlushSize() {
|
|
| 99 |
return gzipFlushSize; |
|
| 100 |
} |
|
| 101 |
|
|
| 102 |
public void setGzipFlushSize(final int gzipFlushSize) {
|
|
| 103 |
this.gzipFlushSize = gzipFlushSize; |
|
| 104 |
} |
|
| 105 |
|
|
| 106 |
public String getTsvFields() {
|
|
| 107 |
return tsvFields; |
|
| 108 |
} |
|
| 109 |
|
|
| 110 |
public void setTsvFields(final String tsvFields) {
|
|
| 111 |
this.tsvFields = tsvFields; |
|
| 112 |
} |
|
| 113 |
|
|
| 114 |
public Resource getProjectsFundingQueryTemplate() {
|
|
| 115 |
return projectsFundingQueryTemplate; |
|
| 116 |
} |
|
| 117 |
|
|
| 118 |
public void setProjectsFundingQueryTemplate(final Resource projectsFundingQueryTemplate) {
|
|
| 119 |
this.projectsFundingQueryTemplate = projectsFundingQueryTemplate; |
|
| 120 |
} |
|
| 121 |
|
|
| 122 |
public Resource getDspaceTemplate() {
|
|
| 123 |
return dspaceTemplate; |
|
| 124 |
} |
|
| 125 |
|
|
| 126 |
public void setDspaceTemplate(final Resource dspaceTemplate) {
|
|
| 127 |
this.dspaceTemplate = dspaceTemplate; |
|
| 128 |
} |
|
| 129 |
|
|
| 130 |
public Resource getDspaceHeadTemplate() {
|
|
| 131 |
return dspaceHeadTemplate; |
|
| 132 |
} |
|
| 133 |
|
|
| 134 |
public void setDspaceHeadTemplate(final Resource dspaceHeadTemplate) {
|
|
| 135 |
this.dspaceHeadTemplate = dspaceHeadTemplate; |
|
| 136 |
} |
|
| 137 |
|
|
| 138 |
public Resource getDspaceTailTemplate() {
|
|
| 139 |
return dspaceTailTemplate; |
|
| 140 |
} |
|
| 141 |
|
|
| 142 |
public void setDspaceTailTemplate(final Resource dspaceTailTemplate) {
|
|
| 143 |
this.dspaceTailTemplate = dspaceTailTemplate; |
|
| 144 |
} |
|
| 145 |
|
|
| 146 |
public Resource getEprintsTemplate() {
|
|
| 147 |
return eprintsTemplate; |
|
| 148 |
} |
|
| 149 |
|
|
| 150 |
public void setEprintsTemplate(final Resource eprintsTemplate) {
|
|
| 151 |
this.eprintsTemplate = eprintsTemplate; |
|
| 152 |
} |
|
| 153 |
} |
|
| 154 |
|
|
| 155 |
public static class Jdbc {
|
|
| 156 |
|
|
| 157 |
// JDBC |
|
| 158 |
@Value("${spring.datasource.driverClassName}")
|
|
| 159 |
private String driverClassName; |
|
| 160 |
|
|
| 161 |
private String url; |
|
| 162 |
private String user; |
|
| 163 |
private String pwd; |
|
| 164 |
private int minIdle; |
|
| 165 |
private int maxidle; |
|
| 166 |
private int maxRows; |
|
| 167 |
|
|
| 168 |
public String getDriverClassName() {
|
|
| 169 |
return driverClassName; |
|
| 170 |
} |
|
| 171 |
|
|
| 172 |
public String getUrl() {
|
|
| 173 |
return url; |
|
| 174 |
} |
|
| 175 |
|
|
| 176 |
public void setUrl(final String url) {
|
|
| 177 |
this.url = url; |
|
| 178 |
} |
|
| 179 |
|
|
| 180 |
public String getUser() {
|
|
| 181 |
return user; |
|
| 182 |
} |
|
| 183 |
|
|
| 184 |
public void setUser(final String user) {
|
|
| 185 |
this.user = user; |
|
| 186 |
} |
|
| 187 |
|
|
| 188 |
public String getPwd() {
|
|
| 189 |
return pwd; |
|
| 190 |
} |
|
| 191 |
|
|
| 192 |
public void setPwd(final String pwd) {
|
|
| 193 |
this.pwd = pwd; |
|
| 194 |
} |
|
| 195 |
|
|
| 196 |
public int getMinIdle() {
|
|
| 197 |
return minIdle; |
|
| 198 |
} |
|
| 199 |
|
|
| 200 |
public void setMinIdle(final int minIdle) {
|
|
| 201 |
this.minIdle = minIdle; |
|
| 202 |
} |
|
| 203 |
|
|
| 204 |
public int getMaxidle() {
|
|
| 205 |
return maxidle; |
|
| 206 |
} |
|
| 207 |
|
|
| 208 |
public void setMaxidle(final int maxidle) {
|
|
| 209 |
this.maxidle = maxidle; |
|
| 210 |
} |
|
| 211 |
|
|
| 212 |
public int getMaxRows() {
|
|
| 213 |
return maxRows; |
|
| 214 |
} |
|
| 215 |
|
|
| 216 |
public void setMaxRows(final int maxRows) {
|
|
| 217 |
this.maxRows = maxRows; |
|
| 218 |
} |
|
| 219 |
} |
|
| 220 |
|
|
| 221 |
public static class Swagger {
|
|
| 222 |
private String apiTitle; |
|
| 223 |
private String apiDescription; |
|
| 224 |
private String apiLicense; |
|
| 225 |
private String apiLicenseUrl; |
|
| 226 |
private String apiContactName; |
|
| 227 |
private String apiContactUrl; |
|
| 228 |
private String apiContactEmail; |
|
| 229 |
|
|
| 230 |
public String getApiTitle() {
|
|
| 231 |
return apiTitle; |
|
| 232 |
} |
|
| 233 |
|
|
| 234 |
public void setApiTitle(final String apiTitle) {
|
|
| 235 |
this.apiTitle = apiTitle; |
|
| 236 |
} |
|
| 237 |
|
|
| 238 |
public String getApiDescription() {
|
|
| 239 |
return apiDescription; |
|
| 240 |
} |
|
| 241 |
|
|
| 242 |
public void setApiDescription(final String apiDescription) {
|
|
| 243 |
this.apiDescription = apiDescription; |
|
| 244 |
} |
|
| 245 |
|
|
| 246 |
public String getApiLicense() {
|
|
| 247 |
return apiLicense; |
|
| 248 |
} |
|
| 249 |
|
|
| 250 |
public void setApiLicense(final String apiLicense) {
|
|
| 251 |
this.apiLicense = apiLicense; |
|
| 252 |
} |
|
| 253 |
|
|
| 254 |
public String getApiLicenseUrl() {
|
|
| 255 |
return apiLicenseUrl; |
|
| 256 |
} |
|
| 257 |
|
|
| 258 |
public void setApiLicenseUrl(final String apiLicenseUrl) {
|
|
| 259 |
this.apiLicenseUrl = apiLicenseUrl; |
|
| 260 |
} |
|
| 261 |
|
|
| 262 |
public String getApiContactName() {
|
|
| 263 |
return apiContactName; |
|
| 264 |
} |
|
| 265 |
|
|
| 266 |
public void setApiContactName(final String apiContactName) {
|
|
| 267 |
this.apiContactName = apiContactName; |
|
| 268 |
} |
|
| 269 |
|
|
| 270 |
public String getApiContactUrl() {
|
|
| 271 |
return apiContactUrl; |
|
| 272 |
} |
|
| 273 |
|
|
| 274 |
public void setApiContactUrl(final String apiContactUrl) {
|
|
| 275 |
this.apiContactUrl = apiContactUrl; |
|
| 276 |
} |
|
| 277 |
|
|
| 278 |
public String getApiContactEmail() {
|
|
| 279 |
return apiContactEmail; |
|
| 280 |
} |
|
| 281 |
|
|
| 282 |
public void setApiContactEmail(final String apiContactEmail) {
|
|
| 283 |
this.apiContactEmail = apiContactEmail; |
|
| 284 |
} |
|
| 285 |
} |
|
| 286 |
|
|
| 287 |
public ClassPathResource getFindSolrIndexUrl() {
|
|
| 288 |
return findSolrIndexUrl; |
|
| 289 |
} |
|
| 290 |
|
|
| 291 |
public void setFindSolrIndexUrl(final ClassPathResource findSolrIndexUrl) {
|
|
| 292 |
this.findSolrIndexUrl = findSolrIndexUrl; |
|
| 293 |
} |
|
| 294 |
|
|
| 295 |
public ClassPathResource getFindIndexDsInfo() {
|
|
| 296 |
return findIndexDsInfo; |
|
| 297 |
} |
|
| 298 |
|
|
| 299 |
public void setFindIndexDsInfo(final ClassPathResource findIndexDsInfo) {
|
|
| 300 |
this.findIndexDsInfo = findIndexDsInfo; |
|
| 301 |
} |
|
| 302 |
|
|
| 303 |
public String getIsLookupUrl() {
|
|
| 304 |
return isLookupUrl; |
|
| 305 |
} |
|
| 306 |
|
|
| 307 |
public void setIsLookupUrl(final String isLookupUrl) {
|
|
| 308 |
this.isLookupUrl = isLookupUrl; |
|
| 309 |
} |
|
| 310 |
|
|
| 311 |
public int getRequestWorkers() {
|
|
| 312 |
return requestWorkers; |
|
| 313 |
} |
|
| 314 |
|
|
| 315 |
public void setRequestWorkers(final int requestWorkers) {
|
|
| 316 |
this.requestWorkers = requestWorkers; |
|
| 317 |
} |
|
| 318 |
|
|
| 319 |
public int getRequestTimeout() {
|
|
| 320 |
return requestTimeout; |
|
| 321 |
} |
|
| 322 |
|
|
| 323 |
public void setRequestTimeout(final int requestTimeout) {
|
|
| 324 |
this.requestTimeout = requestTimeout; |
|
| 325 |
} |
|
| 326 |
|
|
| 327 |
public Datasource getDatasource() {
|
|
| 328 |
return datasource; |
|
| 329 |
} |
|
| 330 |
|
|
| 331 |
public void setDatasource(final Datasource datasource) {
|
|
| 332 |
this.datasource = datasource; |
|
| 333 |
} |
|
| 334 |
|
|
| 335 |
public Project getProject() {
|
|
| 336 |
return project; |
|
| 337 |
} |
|
| 338 |
|
|
| 339 |
public void setProject(final Project project) {
|
|
| 340 |
this.project = project; |
|
| 341 |
} |
|
| 342 |
|
|
| 343 |
public Jdbc getJdbc() {
|
|
| 344 |
return jdbc; |
|
| 345 |
} |
|
| 346 |
|
|
| 347 |
public void setJdbc(final Jdbc jdbc) {
|
|
| 348 |
this.jdbc = jdbc; |
|
| 349 |
} |
|
| 350 |
|
|
| 351 |
public Swagger getSwagger() {
|
|
| 352 |
return swagger; |
|
| 353 |
} |
|
| 354 |
|
|
| 355 |
public void setSwagger(final Swagger swagger) {
|
|
| 356 |
this.swagger = swagger; |
|
| 357 |
} |
|
| 358 |
} |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/main/java/eu/dnetlib/openaire/exporter/project/ProjectQueryParamsFactory.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.openaire.exporter.project; |
|
| 2 |
|
|
| 3 |
import javax.servlet.http.HttpServletRequest; |
|
| 4 |
|
|
| 5 |
import org.springframework.stereotype.Component; |
|
| 6 |
|
|
| 7 |
@Component |
|
| 8 |
public class ProjectQueryParamsFactory {
|
|
| 9 |
|
|
| 10 |
private static final String BASE_PATH = "/openaire/export/"; |
|
| 11 |
private static final String NO_FILTER = "ALL"; |
|
| 12 |
|
|
| 13 |
public ProjectQueryParams generateParams(final HttpServletRequest request, |
|
| 14 |
final String startFrom, |
|
| 15 |
final String startUntil, |
|
| 16 |
final String endFrom, |
|
| 17 |
final String endUntil) {
|
|
| 18 |
ProjectQueryParams params = new ProjectQueryParams(); |
|
| 19 |
|
|
| 20 |
String[] arr = request.getServletPath().replace(BASE_PATH, "").split("\\/");
|
|
| 21 |
if (arr.length != 5) throw new IllegalArgumentException("Invalid url");
|
|
| 22 |
|
|
| 23 |
params.setFundingProgramme(arr[0]); |
|
| 24 |
String stream = NO_FILTER.equals(arr[1]) ? null : arr[1]; |
|
| 25 |
String substream = NO_FILTER.equals(arr[2]) ? null : arr[2]; |
|
| 26 |
if (substream == null) {
|
|
| 27 |
params.setFundingPath(stream); |
|
| 28 |
} else {
|
|
| 29 |
if (stream == null) {
|
|
| 30 |
stream = "%"; |
|
| 31 |
} |
|
| 32 |
params.setFundingPath(stream + "::" + substream); |
|
| 33 |
} |
|
| 34 |
// params.setSpecificProgramme(NO_FILTER.equals(arr[1]) ? null : arr[1]); |
|
| 35 |
// params.setSubdivision(NO_FILTER.equals(arr[2]) ? null : arr[2]); |
|
| 36 |
// NB: arr[3] should be 'projects' |
|
| 37 |
// NB: arr[4] should be '[file].do' |
|
| 38 |
params.setStartFrom(startFrom); |
|
| 39 |
params.setStartUntil(startUntil); |
|
| 40 |
params.setEndFrom(endFrom); |
|
| 41 |
params.setEndUntil(endUntil); |
|
| 42 |
|
|
| 43 |
return params; |
|
| 44 |
} |
|
| 45 |
} |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/main/java/eu/dnetlib/openaire/exporter/project/ProjectsController.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.openaire.exporter.project; |
|
| 2 |
|
|
| 3 |
import java.io.BufferedOutputStream; |
|
| 4 |
import java.io.IOException; |
|
| 5 |
import java.sql.SQLException; |
|
| 6 |
import java.text.SimpleDateFormat; |
|
| 7 |
import java.util.Date; |
|
| 8 |
import java.util.Map; |
|
| 9 |
import javax.servlet.ServletResponse; |
|
| 10 |
import javax.servlet.http.HttpServletRequest; |
|
| 11 |
import javax.servlet.http.HttpServletResponse; |
|
| 12 |
|
|
| 13 |
import com.google.common.xml.XmlEscapers; |
|
| 14 |
import eu.dnetlib.OpenaireExporterConfig; |
|
| 15 |
import eu.dnetlib.OpenaireExporterConfig.Project; |
|
| 16 |
import eu.dnetlib.common.rmi.DNetRestDocumentation; |
|
| 17 |
import org.antlr.stringtemplate.StringTemplate; |
|
| 18 |
import org.apache.commons.io.IOUtils; |
|
| 19 |
import org.apache.commons.lang3.StringUtils; |
|
| 20 |
import org.apache.commons.lang3.exception.ExceptionUtils; |
|
| 21 |
import org.apache.commons.logging.Log; |
|
| 22 |
import org.apache.commons.logging.LogFactory; |
|
| 23 |
import org.springframework.beans.factory.annotation.Autowired; |
|
| 24 |
import org.springframework.core.io.Resource; |
|
| 25 |
import org.springframework.http.HttpStatus; |
|
| 26 |
import org.springframework.stereotype.Controller; |
|
| 27 |
import org.springframework.web.bind.annotation.*; |
|
| 28 |
|
|
| 29 |
@Controller |
|
| 30 |
@DNetRestDocumentation |
|
| 31 |
public class ProjectsController {
|
|
| 32 |
|
|
| 33 |
private static final Log log = LogFactory.getLog(ProjectsController.class); // NOPMD by marko on 11/24/08 5:02 PM |
|
| 34 |
|
|
| 35 |
public final static String UTF8 = "UTF-8"; |
|
| 36 |
|
|
| 37 |
@Autowired |
|
| 38 |
private OpenaireExporterConfig config; |
|
| 39 |
|
|
| 40 |
@Autowired |
|
| 41 |
private JdbcApiDao dao; |
|
| 42 |
|
|
| 43 |
@Autowired |
|
| 44 |
private ProjectQueryParamsFactory projectQueryParamsFactory; |
|
| 45 |
|
|
| 46 |
@RequestMapping(value = "/openaire/export/**/project/dspace.do", method = RequestMethod.GET) |
|
| 47 |
void processDspace(final HttpServletRequest request, final ServletResponse response, |
|
| 48 |
@RequestParam(value = "startFrom", required = false) final String startFrom, |
|
| 49 |
@RequestParam(value = "startUntil", required = false) final String startUntil, |
|
| 50 |
@RequestParam(value = "endFrom", required = false) final String endFrom, |
|
| 51 |
@RequestParam(value = "endUntil", required = false) final String endUntil) throws Exception {
|
|
| 52 |
|
|
| 53 |
final Project conf = config.getProject(); |
|
| 54 |
|
|
| 55 |
final ProjectQueryParams params = projectQueryParamsFactory.generateParams(request, startFrom, startUntil, endFrom, endUntil); |
|
| 56 |
final StringTemplate headSt = new StringTemplate(IOUtils.toString(conf.getDspaceHeadTemplate().getInputStream(), UTF8)); |
|
| 57 |
|
|
| 58 |
headSt.setAttribute("fundingProgramme", params.getFundingProgramme());
|
|
| 59 |
|
|
| 60 |
final StringTemplate tailSt = new StringTemplate(IOUtils.toString(conf.getDspaceTailTemplate().getInputStream(), UTF8)); |
|
| 61 |
|
|
| 62 |
response.setContentType("text/xml");
|
|
| 63 |
doProcess(response, params, headSt.toString(), conf.getDspaceTemplate(), tailSt.toString(), s -> XmlEscapers.xmlContentEscaper().escape(oneLiner(s))); |
|
| 64 |
} |
|
| 65 |
|
|
| 66 |
@RequestMapping(value = "/openaire/export/**/project/eprints.do", method = RequestMethod.GET) |
|
| 67 |
void processEprints(final HttpServletRequest request, final ServletResponse response, |
|
| 68 |
@RequestParam(value = "startFrom", required = false) final String startFrom, |
|
| 69 |
@RequestParam(value = "startUntil", required = false) final String startUntil, |
|
| 70 |
@RequestParam(value = "endFrom", required = false) final String endFrom, |
|
| 71 |
@RequestParam(value = "endUntil", required = false) final String endUntil) throws Exception {
|
|
| 72 |
|
|
| 73 |
final ProjectQueryParams params = projectQueryParamsFactory.generateParams(request, startFrom, startUntil, endFrom, endUntil); |
|
| 74 |
response.setContentType("text/html");
|
|
| 75 |
doProcess(response, params, null, config.getProject().getEprintsTemplate(), null, s -> oneLiner(s)); |
|
| 76 |
} |
|
| 77 |
|
|
| 78 |
private String oneLiner(final String s) {
|
|
| 79 |
return StringUtils.isNotBlank(s) ? s.replaceAll("\\n", " ").trim() : "";
|
|
| 80 |
} |
|
| 81 |
|
|
| 82 |
private void doProcess( |
|
| 83 |
final ServletResponse response, |
|
| 84 |
final ProjectQueryParams params, |
|
| 85 |
final String head, final Resource projectTemplate, final String tail, |
|
| 86 |
final ValueCleaner cleaner) throws IOException, SQLException {
|
|
| 87 |
|
|
| 88 |
final StringTemplate st = new StringTemplate(IOUtils.toString(projectTemplate.getInputStream(), UTF8)); |
|
| 89 |
dao.streamProjects(obtainQuery(params), new BufferedOutputStream(response.getOutputStream()), head, st, tail, cleaner); |
|
| 90 |
} |
|
| 91 |
|
|
| 92 |
@RequestMapping(value = "/openaire/export/project2tsv.do", method = RequestMethod.GET) |
|
| 93 |
void processTsv(final HttpServletRequest request, final HttpServletResponse response, |
|
| 94 |
@RequestParam(value = "funding", required = true) final String funding, |
|
| 95 |
@RequestParam(value = "article293", required = false) final Boolean article293) throws Exception {
|
|
| 96 |
|
|
| 97 |
final String fundingPrefix = getFundingPrefix(funding, null); |
|
| 98 |
|
|
| 99 |
final String date = new SimpleDateFormat("yyyyMMdd").format(new Date());
|
|
| 100 |
final String filename = "projects_" + funding + "_" + date + ".tsv"; |
|
| 101 |
response.setContentType("text/tab-separated-values");
|
|
| 102 |
response.setHeader("Content-Disposition", "attachment; filename=\"" + filename + ".zip\"");
|
|
| 103 |
|
|
| 104 |
dao.processTsvRequest(response.getOutputStream(), article293, fundingPrefix, filename); |
|
| 105 |
} |
|
| 106 |
|
|
| 107 |
@RequestMapping(value = "/openaire/export/streamProjectDetails.do", method = RequestMethod.GET) |
|
| 108 |
void streamProjectDetails(final HttpServletResponse response, |
|
| 109 |
@RequestParam(value = "format", required = true) final String format, |
|
| 110 |
@RequestParam(value = "compress", required = false) final Boolean compress) throws IOException, SQLException {
|
|
| 111 |
|
|
| 112 |
if (compress != null && compress) {
|
|
| 113 |
response.setHeader("Content-Encoding", "gzip");
|
|
| 114 |
} |
|
| 115 |
switch (format) {
|
|
| 116 |
case "csv": |
|
| 117 |
response.setContentType("text/csv");
|
|
| 118 |
break; |
|
| 119 |
case "json": |
|
| 120 |
response.setContentType("text/plain");
|
|
| 121 |
break; |
|
| 122 |
default: throw new IllegalArgumentException("unsupported format: " + format);
|
|
| 123 |
} |
|
| 124 |
|
|
| 125 |
dao.processProjectDetails(response.getOutputStream(), format, compress); |
|
| 126 |
} |
|
| 127 |
|
|
| 128 |
@ExceptionHandler({Exception.class, Throwable.class})
|
|
| 129 |
@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) |
|
| 130 |
public @ResponseBody ErrorMessage handleSqlException(final Exception e) {
|
|
| 131 |
log.debug(e.getMessage(), e); |
|
| 132 |
if (StringUtils.containsIgnoreCase(ExceptionUtils.getRootCauseMessage(e), "Broken pipe")) {
|
|
| 133 |
return null; //socket is closed, cannot return any response |
|
| 134 |
} else {
|
|
| 135 |
return new ErrorMessage(e); |
|
| 136 |
} |
|
| 137 |
} |
|
| 138 |
|
|
| 139 |
public class ErrorMessage {
|
|
| 140 |
|
|
| 141 |
private final String message; |
|
| 142 |
private final String stacktrace; |
|
| 143 |
|
|
| 144 |
public ErrorMessage(final Exception e) {
|
|
| 145 |
this(e.getMessage(), ExceptionUtils.getStackTrace(e)); |
|
| 146 |
} |
|
| 147 |
|
|
| 148 |
public ErrorMessage(final String message, final String stacktrace) {
|
|
| 149 |
this.message = message; |
|
| 150 |
this.stacktrace = stacktrace; |
|
| 151 |
} |
|
| 152 |
|
|
| 153 |
public String getMessage() {
|
|
| 154 |
return this.message; |
|
| 155 |
} |
|
| 156 |
|
|
| 157 |
public String getStacktrace() {
|
|
| 158 |
return this.stacktrace; |
|
| 159 |
} |
|
| 160 |
} |
|
| 161 |
|
|
| 162 |
/** |
|
| 163 |
* Creates the query on the fundingProgramme specified in the given parameters. |
|
| 164 |
* |
|
| 165 |
* @param params |
|
| 166 |
* request parameters |
|
| 167 |
* @return the query string |
|
| 168 |
* @throws IllegalArgumentException |
|
| 169 |
* if the funding program is not recognized |
|
| 170 |
* @throws IOException |
|
| 171 |
* if there are problem loading the query temlate |
|
| 172 |
* @throws IllegalArgumentException |
|
| 173 |
* if the funding program is not recognized |
|
| 174 |
*/ |
|
| 175 |
protected String obtainQuery(final ProjectQueryParams params) throws IllegalArgumentException, IOException {
|
|
| 176 |
String funding = params.getFundingProgramme(); |
|
| 177 |
String suffix = params.getFundingPath(); |
|
| 178 |
|
|
| 179 |
final StringTemplate st = new StringTemplate(IOUtils.toString(config.getProject().getProjectsFundingQueryTemplate().getInputStream(), UTF8)); |
|
| 180 |
st.setAttribute("fundingprefix", getFundingPrefix(funding, suffix));
|
|
| 181 |
String theQuery = setDateParameters(st.toString(), params); |
|
| 182 |
log.debug("Generated query: " + theQuery);
|
|
| 183 |
return theQuery; |
|
| 184 |
} |
|
| 185 |
|
|
| 186 |
private String getFundingPrefix(final String funding, final String suffix) {
|
|
| 187 |
final Map<String, String> fundingIds = dao.readFundingpathIds(); |
|
| 188 |
if (!fundingIds.containsKey(funding.toUpperCase())) {
|
|
| 189 |
throw new IllegalArgumentException("invalid funding " + funding);
|
|
| 190 |
} |
|
| 191 |
String fundingPrefix = fundingIds.get(funding.toUpperCase()); |
|
| 192 |
return StringUtils.isBlank(suffix) ? fundingPrefix : fundingPrefix + "::" + suffix.toUpperCase(); |
|
| 193 |
} |
|
| 194 |
|
|
| 195 |
private String setDateParameters(final String query, final ProjectQueryParams params) {
|
|
| 196 |
String queryWithDates = query; |
|
| 197 |
if (params.getStartFrom() != null) {
|
|
| 198 |
queryWithDates += " AND startdate >= '" + params.getStartFrom() + "'"; |
|
| 199 |
} |
|
| 200 |
if (params.getStartUntil() != null) {
|
|
| 201 |
queryWithDates += " AND startdate <= '" + params.getStartUntil() + "'"; |
|
| 202 |
} |
|
| 203 |
if (params.getEndFrom() != null) {
|
|
| 204 |
queryWithDates += " AND enddate >= '" + params.getEndFrom() + "'"; |
|
| 205 |
} |
|
| 206 |
if (params.getEndUntil() != null) {
|
|
| 207 |
queryWithDates += " AND enddate <= '" + params.getEndUntil() + "'"; |
|
| 208 |
} |
|
| 209 |
return queryWithDates; |
|
| 210 |
} |
|
| 211 |
|
|
| 212 |
} |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/main/java/eu/dnetlib/openaire/exporter/project/repository/ProjectDetailsRepository.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.openaire.exporter.project.repository; |
|
| 2 |
|
|
| 3 |
import eu.dnetlib.openaire.exporter.model.project.ProjectDetails; |
|
| 4 |
import org.springframework.data.repository.PagingAndSortingRepository; |
|
| 5 |
import org.springframework.stereotype.Repository; |
|
| 6 |
|
|
| 7 |
/** |
|
| 8 |
* Created by claudio on 04/07/2017. |
|
| 9 |
*/ |
|
| 10 |
@Repository |
|
| 11 |
public interface ProjectDetailsRepository extends PagingAndSortingRepository<ProjectDetails, String> {
|
|
| 12 |
|
|
| 13 |
} |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/main/java/eu/dnetlib/openaire/exporter/project/repository/ProjectApiRepository.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.openaire.exporter.project.repository; |
|
| 2 |
|
|
| 3 |
import eu.dnetlib.openaire.exporter.model.project.ProjectApi; |
|
| 4 |
import eu.dnetlib.openaire.exporter.model.project.ProjectTsv; |
|
| 5 |
import org.springframework.data.repository.PagingAndSortingRepository; |
|
| 6 |
import org.springframework.stereotype.Repository; |
|
| 7 |
|
|
| 8 |
/** |
|
| 9 |
* Created by claudio on 06/07/2017. |
|
| 10 |
*/ |
|
| 11 |
@Repository |
|
| 12 |
public interface ProjectApiRepository extends PagingAndSortingRepository<ProjectApi, Integer> {
|
|
| 13 |
|
|
| 14 |
Iterable<ProjectTsv> findByFundingpathidStartingWith(String fundingpathid); |
|
| 15 |
|
|
| 16 |
} |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/main/java/eu/dnetlib/openaire/exporter/project/repository/ProjectTsvRepository.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.openaire.exporter.project.repository; |
|
| 2 |
|
|
| 3 |
import eu.dnetlib.openaire.exporter.model.project.ProjectTsv; |
|
| 4 |
import org.springframework.data.repository.PagingAndSortingRepository; |
|
| 5 |
import org.springframework.stereotype.Repository; |
|
| 6 |
|
|
| 7 |
/** |
|
| 8 |
* Created by claudio on 04/07/2017. |
|
| 9 |
*/ |
|
| 10 |
@Repository |
|
| 11 |
public interface ProjectTsvRepository extends PagingAndSortingRepository<ProjectTsv, String> {
|
|
| 12 |
|
|
| 13 |
Iterable<ProjectTsv> findByFundingpathidStartingWithOrderByAcronym(String fundingpathid); |
|
| 14 |
|
|
| 15 |
Iterable<ProjectTsv> findByFundingpathidStartingWithAndOaMandateForDatasetsOrderByAcronym(String fundingpathid, boolean article293); |
|
| 16 |
|
|
| 17 |
} |
|
| modules/dnet-openaire-exporter/tags/dnet-openaire-exporter-1.0.0/src/main/java/eu/dnetlib/openaire/exporter/project/JdbcApiDao.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.openaire.exporter.project; |
|
| 2 |
|
|
| 3 |
import java.io.BufferedOutputStream; |
|
| 4 |
import java.io.IOException; |
|
| 5 |
import java.io.OutputStream; |
|
| 6 |
import java.nio.charset.Charset; |
|
| 7 |
import java.sql.Connection; |
|
| 8 |
import java.sql.PreparedStatement; |
|
| 9 |
import java.sql.ResultSet; |
|
| 10 |
import java.sql.SQLException; |
|
| 11 |
import java.time.Duration; |
|
| 12 |
import java.time.LocalDateTime; |
|
| 13 |
import java.util.List; |
|
| 14 |
import java.util.Map; |
|
| 15 |
import java.util.Set; |
|
| 16 |
import java.util.concurrent.atomic.AtomicInteger; |
|
| 17 |
import java.util.zip.GZIPOutputStream; |
|
| 18 |
import java.util.zip.ZipEntry; |
|
| 19 |
import java.util.zip.ZipOutputStream; |
|
| 20 |
|
|
| 21 |
import com.google.common.base.Joiner; |
|
| 22 |
import com.google.common.base.Splitter; |
|
| 23 |
import com.google.common.collect.Maps; |
|
| 24 |
import com.google.common.collect.Sets; |
|
| 25 |
import eu.dnetlib.OpenaireExporterConfig; |
|
| 26 |
import eu.dnetlib.openaire.exporter.datasource.ApiException; |
|
| 27 |
import eu.dnetlib.openaire.exporter.model.project.Project; |
|
| 28 |
import eu.dnetlib.openaire.exporter.model.project.ProjectTsv; |
|
| 29 |
import eu.dnetlib.openaire.exporter.project.repository.ProjectDetailsRepository; |
|
| 30 |
import eu.dnetlib.openaire.exporter.project.repository.ProjectTsvRepository; |
|
| 31 |
import org.antlr.stringtemplate.StringTemplate; |
|
| 32 |
import org.apache.commons.dbcp2.BasicDataSource; |
|
| 33 |
import org.apache.commons.lang3.StringUtils; |
|
| 34 |
import org.apache.commons.logging.Log; |
|
| 35 |
import org.apache.commons.logging.LogFactory; |
|
| 36 |
import org.springframework.beans.factory.annotation.Autowired; |
|
| 37 |
import org.springframework.stereotype.Component; |
|
| 38 |
|
|
| 39 |
/** |
|
| 40 |
* Created by claudio on 20/09/16. |
|
| 41 |
*/ |
|
| 42 |
@Component |
|
| 43 |
public class JdbcApiDao {
|
|
| 44 |
|
|
| 45 |
public static final Charset UTF8 = Charset.forName("UTF-8");
|
|
| 46 |
|
|
| 47 |
private static final Log log = LogFactory.getLog(JdbcApiDao.class); |
|
| 48 |
|
|
| 49 |
@Autowired |
|
| 50 |
private OpenaireExporterConfig config; |
|
| 51 |
|
|
| 52 |
@Autowired |
|
| 53 |
private BasicDataSource projectApiDataSource; |
|
| 54 |
|
|
| 55 |
@Autowired |
|
| 56 |
private ProjectDetailsRepository projectDetailsRepository; |
|
| 57 |
|
|
| 58 |
@Autowired |
|
| 59 |
private ProjectTsvRepository projectTsvRepository; |
|
| 60 |
|
|
| 61 |
|
|
| 62 |
public Map<String, String> readFundingpathIds() {
|
|
| 63 |
|
|
| 64 |
log.debug("loading funding ids");
|
|
| 65 |
final String sql = "SELECT id FROM fundingpaths"; |
|
| 66 |
final Set<String> ids = Sets.newHashSet(); |
|
| 67 |
try (final Connection con = getConn(); final PreparedStatement stm = getStm(sql, con); final ResultSet rs = stm.executeQuery()) {
|
|
| 68 |
while (rs.next()) {
|
|
| 69 |
ids.add(rs.getString("id"));
|
|
| 70 |
} |
|
| 71 |
} catch (SQLException e) {
|
|
| 72 |
throw new RuntimeException(e); |
|
| 73 |
} |
|
| 74 |
log.debug(String.format("loaded %s funding ids", ids.size()));
|
|
| 75 |
|
|
| 76 |
final Map<String, String> res = Maps.newHashMap(); |
|
| 77 |
final Splitter sp = Splitter.on("::").trimResults();
|
|
| 78 |
ids.stream() |
|
| 79 |
.filter(s -> sp.splitToList(s).size() < 3) |
|
| 80 |
.forEach(s -> res.put(StringUtils.substringAfterLast(s, "::").toUpperCase(), s)); |
|
| 81 |
|
|
| 82 |
res.put("FP7", "ec__________::EC::FP7");
|
|
| 83 |
res.put("H2020", "ec__________::EC::H2020");
|
|
| 84 |
log.debug(String.format("processed %s funding ids", res.size()));
|
|
| 85 |
res.forEach((k,v) -> log.debug(String.format("%s : '%s'", k, v)));
|
|
| 86 |
return res; |
|
| 87 |
} |
|
| 88 |
|
|
| 89 |
public void processProjectDetails(final OutputStream outputStream, String format, Boolean compress) throws IOException {
|
|
| 90 |
final OutputStream out = getOutputStream(outputStream, compress); |
|
| 91 |
try {
|
|
| 92 |
final AtomicInteger i = new AtomicInteger(0); |
|
| 93 |
projectDetailsRepository.findAll().forEach(p -> {
|
|
| 94 |
try {
|
|
| 95 |
switch (format) {
|
|
| 96 |
case "csv": |
|
| 97 |
out.write(p.asCSV().getBytes(UTF8)); |
|
| 98 |
break; |
|
| 99 |
case "json": |
|
| 100 |
out.write(p.asJson().getBytes(UTF8)); |
|
| 101 |
break; |
|
| 102 |
} |
|
| 103 |
i.incrementAndGet(); |
|
| 104 |
if (i.intValue() % config.getProject().getGzipFlushSize() == 0) {
|
|
| 105 |
log.debug("flushing output stream");
|
|
| 106 |
out.flush(); |
|
| 107 |
} |
|
| 108 |
} catch (IOException e) {
|
|
| 109 |
throw new RuntimeException(e); |
|
| 110 |
} |
|
| 111 |
} |
|
| 112 |
); |
|
| 113 |
} finally {
|
|
| 114 |
if (out instanceof GZIPOutputStream) {
|
|
| 115 |
((GZIPOutputStream) out).finish(); |
|
| 116 |
} |
|
| 117 |
out.close(); |
|
| 118 |
} |
|
| 119 |
} |
|
| 120 |
|
|
| 121 |
private OutputStream getOutputStream(final OutputStream outputStream, final Boolean compress) throws IOException {
|
|
| 122 |
if (compress != null && compress) {
|
|
| 123 |
return new GZIPOutputStream(outputStream); |
|
| 124 |
} |
|
| 125 |
return outputStream; |
|
| 126 |
} |
|
| 127 |
|
|
| 128 |
public void processTsvRequest(final OutputStream outputStream, final Boolean article293, final String fundingPrefix, final String filename) throws ApiException {
|
|
| 129 |
try(final ZipOutputStream out = new ZipOutputStream(new BufferedOutputStream(outputStream))) {
|
|
| 130 |
out.putNextEntry(new ZipEntry(filename)); |
|
| 131 |
writeTsvLine(out, Splitter.on(",").trimResults().splitToList(config.getProject().getTsvFields()));
|
|
| 132 |
queryForTsv(fundingPrefix, article293).forEach(p -> {
|
|
| 133 |
try {
|
|
| 134 |
writeTsvLine(out, p.asList()); |
|
| 135 |
} catch (IOException e) {
|
|
| 136 |
throw new RuntimeException(e); |
|
| 137 |
} |
|
| 138 |
}); |
|
| 139 |
} catch (Throwable e) {
|
|
| 140 |
throw new RuntimeException("Error processing the request", e);
|
|
| 141 |
} |
|
| 142 |
} |
|
| 143 |
|
|
| 144 |
private void writeTsvLine(final ZipOutputStream out, final List<String> s) throws IOException {
|
|
| 145 |
out.write(Joiner.on('\t').useForNull("").join(s).getBytes(UTF8));
|
|
| 146 |
out.write('\n');
|
|
| 147 |
} |
|
| 148 |
|
|
| 149 |
private Iterable<ProjectTsv> queryForTsv(final String fundingPrefix, final Boolean article293) {
|
|
| 150 |
log.debug(String.format("fundingPrefix:'%s' and article293:'%s'", fundingPrefix, article293));
|
|
| 151 |
if (article293 != null) {
|
|
| 152 |
return projectTsvRepository.findByFundingpathidStartingWithAndOaMandateForDatasetsOrderByAcronym(fundingPrefix, article293); |
|
| 153 |
} else {
|
|
| 154 |
return projectTsvRepository.findByFundingpathidStartingWithOrderByAcronym(fundingPrefix); |
|
| 155 |
} |
|
| 156 |
} |
|
| 157 |
|
|
| 158 |
public void streamProjects(final String sql, final OutputStream out, |
|
| 159 |
final String head, final StringTemplate projectTemplate, final String tail, |
|
| 160 |
final ValueCleaner cleaner) throws IOException, SQLException {
|
|
| 161 |
|
|
| 162 |
if (log.isDebugEnabled()) {
|
|
| 163 |
log.debug("Thread " + Thread.currentThread().getId() + " begin");
|
|
| 164 |
} |
|
| 165 |
final LocalDateTime start = LocalDateTime.now(); |
|
| 166 |
|
|
| 167 |
if (StringUtils.isNotBlank(head)) {
|
|
| 168 |
out.write(head.getBytes(UTF8)); |
|
| 169 |
} |
|
| 170 |
|
|
| 171 |
try (final Connection con = getConn(); final PreparedStatement stm = getStm(sql, con); final ResultSet rs = stm.executeQuery()) {
|
|
| 172 |
while (rs.next()) {
|
|
| 173 |
final Project p = new Project() |
|
| 174 |
.setFunder(cleaner.clean(rs.getString("funder")))
|
|
| 175 |
.setJurisdiction(cleaner.clean(rs.getString("jurisdiction")))
|
|
| 176 |
.setFundingpathid(cleaner.clean(rs.getString("fundingpathid")))
|
|
| 177 |
.setAcronym(cleaner.clean(rs.getString("acronym")))
|
|
| 178 |
.setTitle(cleaner.clean(rs.getString("title")))
|
|
| 179 |
.setCode(cleaner.clean(rs.getString("code")))
|
|
| 180 |
.setStartdate(cleaner.clean(rs.getString("startdate")))
|
|
| 181 |
.setEnddate(cleaner.clean(rs.getString("enddate")));
|
|
| 182 |
|
|
| 183 |
projectTemplate.reset(); |
|
Also available in: Unified diff
removed wrong tag