Revision 48399
Added by Claudio Atzori about 7 years ago
modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/datasource/clients/JdbcDatasourceDao.java | ||
---|---|---|
1 |
package eu.dnetlib.openaire.exporter.datasource.clients; |
|
2 |
|
|
3 |
import java.io.IOException; |
|
4 |
import java.sql.Connection; |
|
5 |
import java.sql.PreparedStatement; |
|
6 |
import java.sql.ResultSet; |
|
7 |
import java.sql.SQLException; |
|
8 |
import java.util.List; |
|
9 |
|
|
10 |
import com.google.common.collect.Lists; |
|
11 |
import eu.dnetlib.openaire.exporter.datasource.ApiException; |
|
12 |
import org.antlr.stringtemplate.StringTemplate; |
|
13 |
import org.apache.commons.dbcp2.BasicDataSource; |
|
14 |
import org.apache.commons.io.IOUtils; |
|
15 |
import org.apache.commons.logging.Log; |
|
16 |
import org.apache.commons.logging.LogFactory; |
|
17 |
import org.springframework.beans.factory.annotation.Autowired; |
|
18 |
import org.springframework.beans.factory.annotation.Value; |
|
19 |
import org.springframework.core.io.ClassPathResource; |
|
20 |
import org.springframework.core.io.Resource; |
|
21 |
import org.springframework.http.HttpStatus; |
|
22 |
import org.springframework.stereotype.Component; |
|
23 |
|
|
24 |
/** |
|
25 |
* Created by claudio on 25/11/2016. |
|
26 |
*/ |
|
27 |
@Component |
|
28 |
public class JdbcDatasourceDao { |
|
29 |
|
|
30 |
private static final Log log = LogFactory.getLog(JdbcDatasourceDao.class); |
|
31 |
|
|
32 |
@Value("${openaire.datasource.publisher.getsingle.sql}") |
|
33 |
private ClassPathResource datasourceDetailsSql; |
|
34 |
|
|
35 |
@Autowired |
|
36 |
private BasicDataSource dsDataSource; |
|
37 |
|
|
38 |
public List<String> listIds() throws ApiException { |
|
39 |
final List<String> res = Lists.newArrayList(); |
|
40 |
try { |
|
41 |
final String sql = "SELECT id FROM dashboard_ds"; |
|
42 |
|
|
43 |
try (final Connection con = getConn(); final PreparedStatement stm = getStm(sql, con); final ResultSet rs = stm.executeQuery()) { |
|
44 |
while (rs.next()) { |
|
45 |
res.add(rs.getString("id")); |
|
46 |
} |
|
47 |
} |
|
48 |
} catch(SQLException e) { |
|
49 |
throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying datasource ids from database", e); |
|
50 |
} |
|
51 |
return res; |
|
52 |
} |
|
53 |
|
|
54 |
private Connection getConn() throws SQLException { |
|
55 |
final Connection conn = dsDataSource.getConnection(); |
|
56 |
conn.setAutoCommit(false); |
|
57 |
return conn; |
|
58 |
} |
|
59 |
|
|
60 |
private PreparedStatement getStm(final String sql, final Connection con) throws SQLException { |
|
61 |
final PreparedStatement stm = con.prepareStatement(sql); |
|
62 |
stm.setFetchSize(10); |
|
63 |
return stm; |
|
64 |
} |
|
65 |
|
|
66 |
protected String obtainQuery(final Resource template, final String id) throws IllegalArgumentException, IOException { |
|
67 |
final StringTemplate st = new StringTemplate(IOUtils.toString(template.getInputStream())); |
|
68 |
st.setAttribute("id", id); |
|
69 |
final String sql = st.toString(); |
|
70 |
log.debug("Generated query: " + sql); |
|
71 |
return sql; |
|
72 |
} |
|
73 |
|
|
74 |
} |
modules/dnet-openaire-exporter/trunk/src/main/resources/eu/dnetlib/openaire/exporter/sql/projects_tsv.sql.st | ||
---|---|---|
1 |
SELECT |
|
2 |
code, |
|
3 |
acronym, |
|
4 |
title, |
|
5 |
call_identifier, |
|
6 |
startdate, |
|
7 |
enddate, |
|
8 |
oa_mandate_for_publications, |
|
9 |
oa_mandate_for_datasets, |
|
10 |
description, |
|
11 |
legalname, |
|
12 |
country, |
|
13 |
role, |
|
14 |
firstname, |
|
15 |
secondnames, |
|
16 |
|
|
17 |
FROM projects_tsv |
|
18 |
WHERE fundingpathid like '$fundingprefix$%'$filters.keys:{k| AND $k$=$filters.(k)$}$ |
|
19 |
ORDER BY acronym |
modules/dnet-openaire-exporter/trunk/src/test/java/eu/dnetlib/openaire/exporter/project/ProjectsControllerTest.java | ||
---|---|---|
15 | 15 |
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; |
16 | 16 |
|
17 | 17 |
import static org.junit.Assert.assertEquals; |
18 |
|
|
19 |
@SpringBootTest |
|
18 | 20 |
@RunWith(SpringJUnit4ClassRunner.class) |
19 |
//@ContextConfiguration(classes={TestConfiguration.class}) |
|
20 |
@SpringBootTest |
|
21 | 21 |
public class ProjectsControllerTest { |
22 | 22 |
|
23 | 23 |
private static final Log log = LogFactory.getLog(ProjectsControllerTest.class); |
... | ... | |
33 | 33 |
controller = new ProjectsController(); |
34 | 34 |
Resource template = new ClassPathResource(queryTemplate); |
35 | 35 |
|
36 |
controller.setProjectsFundingQueryTemplate(template); |
|
36 |
//TODO reimplement bean injection for testing |
|
37 |
//controller.setProjectsFundingQueryTemplate(template); |
|
37 | 38 |
params = new ProjectQueryParams(); |
38 | 39 |
} |
39 | 40 |
|
modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/swagger/configuration/SwaggerDocumentationConfig.java | ||
---|---|---|
1 | 1 |
package eu.dnetlib.swagger.configuration; |
2 | 2 |
|
3 |
import eu.dnetlib.OpenaireExporterConfig; |
|
4 |
import eu.dnetlib.OpenaireExporterConfig.Swagger; |
|
3 | 5 |
import eu.dnetlib.common.rmi.DNetRestDocumentation; |
6 |
import org.springframework.beans.factory.annotation.Autowired; |
|
4 | 7 |
import org.springframework.beans.factory.annotation.Value; |
5 | 8 |
import org.springframework.context.annotation.Bean; |
6 | 9 |
import org.springframework.context.annotation.Configuration; |
... | ... | |
16 | 19 |
@EnableSwagger2 |
17 | 20 |
public class SwaggerDocumentationConfig { |
18 | 21 |
|
19 |
@Value("${api.rest.contact.name}") |
|
20 |
private String contactName; |
|
21 |
@Value("${api.rest.contact.url}") |
|
22 |
private String contactUrl; |
|
23 |
@Value("${api.rest.contact.email}") |
|
24 |
private String contactEmail; |
|
25 |
@Value("${api.rest.title}") |
|
26 |
private String title; |
|
27 |
@Value("${api.rest.description}") |
|
28 |
private String description; |
|
29 |
@Value("${api.rest.license}") |
|
30 |
private String license; |
|
31 |
@Value("${api.rest.license.url}") |
|
32 |
private String licenseUrl; |
|
22 |
@Autowired |
|
23 |
private OpenaireExporterConfig config; |
|
33 | 24 |
|
34 | 25 |
@Bean |
35 | 26 |
public Docket customImplementation() { |
... | ... | |
43 | 34 |
} |
44 | 35 |
|
45 | 36 |
private ApiInfo apiInfo() { |
37 |
final Swagger swag = config.getSwagger(); |
|
46 | 38 |
return new ApiInfoBuilder() |
47 |
.title(title)
|
|
48 |
.description(description)
|
|
49 |
.license(license)
|
|
50 |
.licenseUrl(licenseUrl)
|
|
39 |
.title(swag.getApiTitle())
|
|
40 |
.description(swag.getApiDescription())
|
|
41 |
.license(swag.getApiLicense())
|
|
42 |
.licenseUrl(swag.getApiLicenseUrl())
|
|
51 | 43 |
.termsOfServiceUrl("") |
52 | 44 |
.version("1.0.0") |
53 |
.contact(new Contact(contactName, contactUrl, contactEmail)) |
|
45 |
.contact(new Contact( |
|
46 |
swag.getApiContactName(), |
|
47 |
swag.getApiContactUrl(), |
|
48 |
swag.getApiContactEmail())) |
|
54 | 49 |
.build(); |
55 | 50 |
} |
56 | 51 |
|
modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/OpenaireExporterConfig.java | ||
---|---|---|
1 | 1 |
package eu.dnetlib; |
2 | 2 |
|
3 |
import com.mongodb.MongoClient; |
|
4 |
import com.mongodb.MongoClientOptions; |
|
5 |
import com.mongodb.ServerAddress; |
|
6 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; |
|
7 |
import org.apache.commons.dbcp2.BasicDataSource; |
|
8 |
import org.apache.cxf.jaxws.JaxWsProxyFactoryBean; |
|
9 | 3 |
import org.springframework.beans.factory.annotation.Value; |
10 |
import org.springframework.context.annotation.Bean; |
|
11 |
import org.springframework.context.annotation.Configuration; |
|
4 |
import org.springframework.boot.context.properties.ConfigurationProperties; |
|
5 |
import org.springframework.core.io.ClassPathResource; |
|
6 |
import org.springframework.core.io.Resource; |
|
7 |
import org.springframework.stereotype.Component; |
|
12 | 8 |
|
13 | 9 |
/** |
14 | 10 |
* Created by Alessia Bardi on 31/03/17. |
15 | 11 |
* |
16 | 12 |
* @author Alessia Bardi |
17 | 13 |
*/ |
18 |
@Configuration |
|
14 |
@Component |
|
15 |
@ConfigurationProperties(prefix = "openaire.exporter") |
|
19 | 16 |
public class OpenaireExporterConfig { |
20 | 17 |
|
21 |
// POSTGRES |
|
22 |
@Value("${spring.datasource.driverClassName}") |
|
23 |
private String driverClassName; |
|
24 |
@Value("${openaire.exporter.jdbc.url}") |
|
25 |
private String jdbcUrl; |
|
26 |
@Value("${openaire.exporter.jdbc.user}") |
|
27 |
private String jdbcUser; |
|
28 |
@Value("${openaire.exporter.jdbc.pwd}") |
|
29 |
private String jdbcPwd; |
|
30 |
@Value("${openaire.exporter.jdbc.minIdle}") |
|
31 |
private int jdbcMinIdle; |
|
32 |
@Value("${openaire.exporter.jdbc.maxIdle}") |
|
33 |
private int jdbcMaxidle; |
|
18 |
// ISLOOKUP |
|
19 |
private ClassPathResource findSolrIndexUrl; |
|
20 |
private ClassPathResource findIndexDsInfo; |
|
21 |
private String isLookupUrl; |
|
22 |
private int requestWorkers = 100; |
|
23 |
private int requestTimeout = 10; |
|
34 | 24 |
|
35 |
// MONGODB |
|
36 |
@Value("${openaire.datasource.publisher.mongodb.host}") |
|
37 |
private String mongodbHost; |
|
38 |
@Value("${openaire.datasource.publisher.mongodb.port}") |
|
39 |
private int mongodbPort; |
|
40 |
@Value("${openaire.datasource.publisher.mongodb.connectionsPerHost}") |
|
41 |
private int connectionsPerHost; |
|
25 |
private Datasource datasource; |
|
26 |
private Project project; |
|
27 |
private Jdbc jdbc; |
|
28 |
private Swagger swagger; |
|
42 | 29 |
|
43 |
@Value("${services.is.lookup.url}") |
|
44 |
private String isLookupUrl; |
|
30 |
public static class Datasource { |
|
31 |
// MONGODB |
|
32 |
private String mongoHost; |
|
33 |
private int mongoPort; |
|
34 |
private String mongoCollectionName; |
|
35 |
private String mongoDbName; |
|
36 |
private int mongoConnectionsPerHost; |
|
37 |
private int mongoQueryLimit; |
|
45 | 38 |
|
46 |
@Bean |
|
47 |
public ISLookUpService getLookUpService() { |
|
48 |
final JaxWsProxyFactoryBean jaxWsProxyFactory = new JaxWsProxyFactoryBean(); |
|
49 |
jaxWsProxyFactory.setServiceClass(ISLookUpService.class); |
|
50 |
jaxWsProxyFactory.setAddress(isLookupUrl); |
|
51 |
return (ISLookUpService) jaxWsProxyFactory.create(); |
|
39 |
public String getMongoHost() { |
|
40 |
return mongoHost; |
|
41 |
} |
|
42 |
|
|
43 |
public void setMongoHost(final String mongoHost) { |
|
44 |
this.mongoHost = mongoHost; |
|
45 |
} |
|
46 |
|
|
47 |
public int getMongoPort() { |
|
48 |
return mongoPort; |
|
49 |
} |
|
50 |
|
|
51 |
public void setMongoPort(final int mongoPort) { |
|
52 |
this.mongoPort = mongoPort; |
|
53 |
} |
|
54 |
|
|
55 |
public String getMongoCollectionName() { |
|
56 |
return mongoCollectionName; |
|
57 |
} |
|
58 |
|
|
59 |
public void setMongoCollectionName(final String mongoCollectionName) { |
|
60 |
this.mongoCollectionName = mongoCollectionName; |
|
61 |
} |
|
62 |
|
|
63 |
public String getMongoDbName() { |
|
64 |
return mongoDbName; |
|
65 |
} |
|
66 |
|
|
67 |
public void setMongoDbName(final String mongoDbName) { |
|
68 |
this.mongoDbName = mongoDbName; |
|
69 |
} |
|
70 |
|
|
71 |
public int getMongoConnectionsPerHost() { |
|
72 |
return mongoConnectionsPerHost; |
|
73 |
} |
|
74 |
|
|
75 |
public void setMongoConnectionsPerHost(final int mongoConnectionsPerHost) { |
|
76 |
this.mongoConnectionsPerHost = mongoConnectionsPerHost; |
|
77 |
} |
|
78 |
|
|
79 |
public int getMongoQueryLimit() { |
|
80 |
return mongoQueryLimit; |
|
81 |
} |
|
82 |
|
|
83 |
public void setMongoQueryLimit(final int mongoQueryLimit) { |
|
84 |
this.mongoQueryLimit = mongoQueryLimit; |
|
85 |
} |
|
52 | 86 |
} |
53 | 87 |
|
54 |
@Bean |
|
55 |
public BasicDataSource getProjectDataSource() { |
|
56 |
return getDatasource(driverClassName, jdbcUrl, jdbcUser, jdbcPwd, jdbcMinIdle, jdbcMaxidle); |
|
88 |
public static class Project { |
|
89 |
|
|
90 |
private int gzipFlushSize; |
|
91 |
private String tsvFields; |
|
92 |
private Resource projectsFundingQueryTemplate; |
|
93 |
private Resource dspaceTemplate; |
|
94 |
private Resource dspaceHeadTemplate; |
|
95 |
private Resource dspaceTailTemplate; |
|
96 |
private Resource eprintsTemplate; |
|
97 |
|
|
98 |
public int getGzipFlushSize() { |
|
99 |
return gzipFlushSize; |
|
100 |
} |
|
101 |
|
|
102 |
public void setGzipFlushSize(final int gzipFlushSize) { |
|
103 |
this.gzipFlushSize = gzipFlushSize; |
|
104 |
} |
|
105 |
|
|
106 |
public String getTsvFields() { |
|
107 |
return tsvFields; |
|
108 |
} |
|
109 |
|
|
110 |
public void setTsvFields(final String tsvFields) { |
|
111 |
this.tsvFields = tsvFields; |
|
112 |
} |
|
113 |
|
|
114 |
public Resource getProjectsFundingQueryTemplate() { |
|
115 |
return projectsFundingQueryTemplate; |
|
116 |
} |
|
117 |
|
|
118 |
public void setProjectsFundingQueryTemplate(final Resource projectsFundingQueryTemplate) { |
|
119 |
this.projectsFundingQueryTemplate = projectsFundingQueryTemplate; |
|
120 |
} |
|
121 |
|
|
122 |
public Resource getDspaceTemplate() { |
|
123 |
return dspaceTemplate; |
|
124 |
} |
|
125 |
|
|
126 |
public void setDspaceTemplate(final Resource dspaceTemplate) { |
|
127 |
this.dspaceTemplate = dspaceTemplate; |
|
128 |
} |
|
129 |
|
|
130 |
public Resource getDspaceHeadTemplate() { |
|
131 |
return dspaceHeadTemplate; |
|
132 |
} |
|
133 |
|
|
134 |
public void setDspaceHeadTemplate(final Resource dspaceHeadTemplate) { |
|
135 |
this.dspaceHeadTemplate = dspaceHeadTemplate; |
|
136 |
} |
|
137 |
|
|
138 |
public Resource getDspaceTailTemplate() { |
|
139 |
return dspaceTailTemplate; |
|
140 |
} |
|
141 |
|
|
142 |
public void setDspaceTailTemplate(final Resource dspaceTailTemplate) { |
|
143 |
this.dspaceTailTemplate = dspaceTailTemplate; |
|
144 |
} |
|
145 |
|
|
146 |
public Resource getEprintsTemplate() { |
|
147 |
return eprintsTemplate; |
|
148 |
} |
|
149 |
|
|
150 |
public void setEprintsTemplate(final Resource eprintsTemplate) { |
|
151 |
this.eprintsTemplate = eprintsTemplate; |
|
152 |
} |
|
57 | 153 |
} |
58 | 154 |
|
59 |
private BasicDataSource getDatasource(String driverClassName, String jdbcUrl, String jdbcUser, String jdbcPwd, int jdbcMinIdle, int jdbcMaxIdle) { |
|
60 |
BasicDataSource d = new BasicDataSource(); |
|
61 |
d.setDriverClassName(driverClassName); |
|
62 |
d.setUrl(jdbcUrl); |
|
63 |
d.setUsername(jdbcUser); |
|
64 |
d.setPassword(jdbcPwd); |
|
65 |
d.setMinIdle(jdbcMinIdle); |
|
66 |
d.setMaxIdle(jdbcMaxIdle); |
|
67 |
return d; |
|
155 |
public static class Jdbc { |
|
156 |
|
|
157 |
// JDBC |
|
158 |
@Value("${spring.datasource.driverClassName}") |
|
159 |
private String driverClassName; |
|
160 |
|
|
161 |
private String url; |
|
162 |
private String user; |
|
163 |
private String pwd; |
|
164 |
private int minIdle; |
|
165 |
private int maxidle; |
|
166 |
private int maxRows; |
|
167 |
|
|
168 |
public String getDriverClassName() { |
|
169 |
return driverClassName; |
|
170 |
} |
|
171 |
|
|
172 |
public String getUrl() { |
|
173 |
return url; |
|
174 |
} |
|
175 |
|
|
176 |
public void setUrl(final String url) { |
|
177 |
this.url = url; |
|
178 |
} |
|
179 |
|
|
180 |
public String getUser() { |
|
181 |
return user; |
|
182 |
} |
|
183 |
|
|
184 |
public void setUser(final String user) { |
|
185 |
this.user = user; |
|
186 |
} |
|
187 |
|
|
188 |
public String getPwd() { |
|
189 |
return pwd; |
|
190 |
} |
|
191 |
|
|
192 |
public void setPwd(final String pwd) { |
|
193 |
this.pwd = pwd; |
|
194 |
} |
|
195 |
|
|
196 |
public int getMinIdle() { |
|
197 |
return minIdle; |
|
198 |
} |
|
199 |
|
|
200 |
public void setMinIdle(final int minIdle) { |
|
201 |
this.minIdle = minIdle; |
|
202 |
} |
|
203 |
|
|
204 |
public int getMaxidle() { |
|
205 |
return maxidle; |
|
206 |
} |
|
207 |
|
|
208 |
public void setMaxidle(final int maxidle) { |
|
209 |
this.maxidle = maxidle; |
|
210 |
} |
|
211 |
|
|
212 |
public int getMaxRows() { |
|
213 |
return maxRows; |
|
214 |
} |
|
215 |
|
|
216 |
public void setMaxRows(final int maxRows) { |
|
217 |
this.maxRows = maxRows; |
|
218 |
} |
|
68 | 219 |
} |
69 | 220 |
|
70 |
@Bean |
|
71 |
public MongoClient getMongoClient() { |
|
72 |
return new MongoClient( |
|
73 |
new ServerAddress(mongodbHost, mongodbPort), |
|
74 |
MongoClientOptions.builder().connectionsPerHost(connectionsPerHost).build()); |
|
221 |
public static class Swagger { |
|
222 |
private String apiTitle; |
|
223 |
private String apiDescription; |
|
224 |
private String apiLicense; |
|
225 |
private String apiLicenseUrl; |
|
226 |
private String apiContactName; |
|
227 |
private String apiContactUrl; |
|
228 |
private String apiContactEmail; |
|
229 |
|
|
230 |
public String getApiTitle() { |
|
231 |
return apiTitle; |
|
232 |
} |
|
233 |
|
|
234 |
public void setApiTitle(final String apiTitle) { |
|
235 |
this.apiTitle = apiTitle; |
|
236 |
} |
|
237 |
|
|
238 |
public String getApiDescription() { |
|
239 |
return apiDescription; |
|
240 |
} |
|
241 |
|
|
242 |
public void setApiDescription(final String apiDescription) { |
|
243 |
this.apiDescription = apiDescription; |
|
244 |
} |
|
245 |
|
|
246 |
public String getApiLicense() { |
|
247 |
return apiLicense; |
|
248 |
} |
|
249 |
|
|
250 |
public void setApiLicense(final String apiLicense) { |
|
251 |
this.apiLicense = apiLicense; |
|
252 |
} |
|
253 |
|
|
254 |
public String getApiLicenseUrl() { |
|
255 |
return apiLicenseUrl; |
|
256 |
} |
|
257 |
|
|
258 |
public void setApiLicenseUrl(final String apiLicenseUrl) { |
|
259 |
this.apiLicenseUrl = apiLicenseUrl; |
|
260 |
} |
|
261 |
|
|
262 |
public String getApiContactName() { |
|
263 |
return apiContactName; |
|
264 |
} |
|
265 |
|
|
266 |
public void setApiContactName(final String apiContactName) { |
|
267 |
this.apiContactName = apiContactName; |
|
268 |
} |
|
269 |
|
|
270 |
public String getApiContactUrl() { |
|
271 |
return apiContactUrl; |
|
272 |
} |
|
273 |
|
|
274 |
public void setApiContactUrl(final String apiContactUrl) { |
|
275 |
this.apiContactUrl = apiContactUrl; |
|
276 |
} |
|
277 |
|
|
278 |
public String getApiContactEmail() { |
|
279 |
return apiContactEmail; |
|
280 |
} |
|
281 |
|
|
282 |
public void setApiContactEmail(final String apiContactEmail) { |
|
283 |
this.apiContactEmail = apiContactEmail; |
|
284 |
} |
|
75 | 285 |
} |
76 | 286 |
|
287 |
public ClassPathResource getFindSolrIndexUrl() { |
|
288 |
return findSolrIndexUrl; |
|
289 |
} |
|
290 |
|
|
291 |
public void setFindSolrIndexUrl(final ClassPathResource findSolrIndexUrl) { |
|
292 |
this.findSolrIndexUrl = findSolrIndexUrl; |
|
293 |
} |
|
294 |
|
|
295 |
public ClassPathResource getFindIndexDsInfo() { |
|
296 |
return findIndexDsInfo; |
|
297 |
} |
|
298 |
|
|
299 |
public void setFindIndexDsInfo(final ClassPathResource findIndexDsInfo) { |
|
300 |
this.findIndexDsInfo = findIndexDsInfo; |
|
301 |
} |
|
302 |
|
|
303 |
public String getIsLookupUrl() { |
|
304 |
return isLookupUrl; |
|
305 |
} |
|
306 |
|
|
307 |
public void setIsLookupUrl(final String isLookupUrl) { |
|
308 |
this.isLookupUrl = isLookupUrl; |
|
309 |
} |
|
310 |
|
|
311 |
public int getRequestWorkers() { |
|
312 |
return requestWorkers; |
|
313 |
} |
|
314 |
|
|
315 |
public void setRequestWorkers(final int requestWorkers) { |
|
316 |
this.requestWorkers = requestWorkers; |
|
317 |
} |
|
318 |
|
|
319 |
public int getRequestTimeout() { |
|
320 |
return requestTimeout; |
|
321 |
} |
|
322 |
|
|
323 |
public void setRequestTimeout(final int requestTimeout) { |
|
324 |
this.requestTimeout = requestTimeout; |
|
325 |
} |
|
326 |
|
|
327 |
public Datasource getDatasource() { |
|
328 |
return datasource; |
|
329 |
} |
|
330 |
|
|
331 |
public void setDatasource(final Datasource datasource) { |
|
332 |
this.datasource = datasource; |
|
333 |
} |
|
334 |
|
|
335 |
public Project getProject() { |
|
336 |
return project; |
|
337 |
} |
|
338 |
|
|
339 |
public void setProject(final Project project) { |
|
340 |
this.project = project; |
|
341 |
} |
|
342 |
|
|
343 |
public Jdbc getJdbc() { |
|
344 |
return jdbc; |
|
345 |
} |
|
346 |
|
|
347 |
public void setJdbc(final Jdbc jdbc) { |
|
348 |
this.jdbc = jdbc; |
|
349 |
} |
|
350 |
|
|
351 |
public Swagger getSwagger() { |
|
352 |
return swagger; |
|
353 |
} |
|
354 |
|
|
355 |
public void setSwagger(final Swagger swagger) { |
|
356 |
this.swagger = swagger; |
|
357 |
} |
|
77 | 358 |
} |
modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/model/project/ProjectApi.java | ||
---|---|---|
1 |
package eu.dnetlib.openaire.exporter.model.project; |
|
2 |
|
|
3 |
import java.sql.Date; |
|
4 |
import java.util.ArrayList; |
|
5 |
import javax.persistence.Entity; |
|
6 |
import javax.persistence.Id; |
|
7 |
import javax.persistence.Table; |
|
8 |
|
|
9 |
import com.fasterxml.jackson.annotation.JsonIgnore; |
|
10 |
import com.google.common.base.Splitter; |
|
11 |
import com.google.common.collect.Lists; |
|
12 |
import io.swagger.annotations.ApiModel; |
|
13 |
import org.apache.commons.lang3.StringUtils; |
|
14 |
|
|
15 |
/** |
|
16 |
* Created by claudio on 20/09/16. |
|
17 |
*/ |
|
18 |
@Entity |
|
19 |
@Table(name = "projects_api") |
|
20 |
@ApiModel(value = "Project api model", description = "Project api model used by DSpace and Eprints exporter") |
|
21 |
public class ProjectApi { |
|
22 |
|
|
23 |
public static final String INFO_EU_REPO_GRANT_AGREEMENT = "info:eu-repo/grantAgreement/"; |
|
24 |
|
|
25 |
@Id |
|
26 |
@JsonIgnore |
|
27 |
private int id; |
|
28 |
|
|
29 |
private String code; |
|
30 |
private String acronym; |
|
31 |
private String title; |
|
32 |
private String funder; |
|
33 |
private String jurisdiction; |
|
34 |
private Date startdate; |
|
35 |
private Date enddate; |
|
36 |
private String fundingpathid; |
|
37 |
|
|
38 |
public ProjectApi() { } |
|
39 |
|
|
40 |
public String getIdnamespace() { |
|
41 |
String res = INFO_EU_REPO_GRANT_AGREEMENT + getFunder()+"/"; |
|
42 |
final String fundingProgram = asFundingProgram(getFundingpathid()); |
|
43 |
if (StringUtils.isNotBlank(fundingProgram)) { |
|
44 |
res += fundingProgram; |
|
45 |
} |
|
46 |
res += "/" + escapeCode(getCode()); |
|
47 |
if (StringUtils.isNotBlank(getJurisdiction())) { |
|
48 |
res += "/" + getJurisdiction(); |
|
49 |
} |
|
50 |
return res; |
|
51 |
} |
|
52 |
|
|
53 |
public String getListLabel() { |
|
54 |
return String.format("for:value:component:_%s_project_id", asFunder(getFunder())); |
|
55 |
} |
|
56 |
|
|
57 |
private String asFunder(final String legalshortname) { |
|
58 |
switch (legalshortname.toLowerCase()) { |
|
59 |
case "ec": |
|
60 |
return asFundingProgram(getFundingpathid()).toLowerCase(); |
|
61 |
default: |
|
62 |
return legalshortname.toLowerCase(); |
|
63 |
} |
|
64 |
} |
|
65 |
|
|
66 |
private String escapeCode(final String code) { |
|
67 |
return replaceSlash(code); |
|
68 |
} |
|
69 |
|
|
70 |
private String asFundingProgram(final String fundingpathid) { |
|
71 |
final ArrayList<String> strings = Lists.newArrayList(Splitter.on("::").split(fundingpathid)); |
|
72 |
if(strings.size() <= 1) throw new IllegalStateException("Unexpected funding id: "+fundingpathid); |
|
73 |
if(strings.size() == 2) return ""; |
|
74 |
else return replaceSlash(strings.get(2)); |
|
75 |
} |
|
76 |
|
|
77 |
private String replaceSlash(final String s) { |
|
78 |
return s.replaceAll("/", "%2F"); |
|
79 |
} |
|
80 |
|
|
81 |
public int getId() { |
|
82 |
return id; |
|
83 |
} |
|
84 |
|
|
85 |
public void setId(final int id) { |
|
86 |
this.id = id; |
|
87 |
} |
|
88 |
|
|
89 |
public String getCode() { |
|
90 |
return code; |
|
91 |
} |
|
92 |
|
|
93 |
public void setCode(final String code) { |
|
94 |
this.code = code; |
|
95 |
} |
|
96 |
|
|
97 |
public String getAcronym() { |
|
98 |
return acronym; |
|
99 |
} |
|
100 |
|
|
101 |
public void setAcronym(final String acronym) { |
|
102 |
this.acronym = acronym; |
|
103 |
} |
|
104 |
|
|
105 |
public String getTitle() { |
|
106 |
return title; |
|
107 |
} |
|
108 |
|
|
109 |
public void setTitle(final String title) { |
|
110 |
this.title = title; |
|
111 |
} |
|
112 |
|
|
113 |
public String getFunder() { |
|
114 |
return funder; |
|
115 |
} |
|
116 |
|
|
117 |
public void setFunder(final String funder) { |
|
118 |
this.funder = funder; |
|
119 |
} |
|
120 |
|
|
121 |
public String getJurisdiction() { |
|
122 |
return jurisdiction; |
|
123 |
} |
|
124 |
|
|
125 |
public void setJurisdiction(final String jurisdiction) { |
|
126 |
this.jurisdiction = jurisdiction; |
|
127 |
} |
|
128 |
|
|
129 |
public Date getStartdate() { |
|
130 |
return startdate; |
|
131 |
} |
|
132 |
|
|
133 |
public void setStartdate(final Date startdate) { |
|
134 |
this.startdate = startdate; |
|
135 |
} |
|
136 |
|
|
137 |
public Date getEnddate() { |
|
138 |
return enddate; |
|
139 |
} |
|
140 |
|
|
141 |
public void setEnddate(final Date enddate) { |
|
142 |
this.enddate = enddate; |
|
143 |
} |
|
144 |
|
|
145 |
public String getFundingpathid() { |
|
146 |
return fundingpathid; |
|
147 |
} |
|
148 |
|
|
149 |
public void setFundingpathid(final String fundingpathid) { |
|
150 |
this.fundingpathid = fundingpathid; |
|
151 |
} |
|
152 |
} |
modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/project/repository/ProjectApiRepository.java | ||
---|---|---|
1 |
package eu.dnetlib.openaire.exporter.project.repository; |
|
2 |
|
|
3 |
import eu.dnetlib.openaire.exporter.model.project.ProjectApi; |
|
4 |
import eu.dnetlib.openaire.exporter.model.project.ProjectTsv; |
|
5 |
import org.springframework.data.repository.PagingAndSortingRepository; |
|
6 |
import org.springframework.stereotype.Repository; |
|
7 |
|
|
8 |
/** |
|
9 |
* Created by claudio on 06/07/2017. |
|
10 |
*/ |
|
11 |
@Repository |
|
12 |
public interface ProjectApiRepository extends PagingAndSortingRepository<ProjectApi, Integer> { |
|
13 |
|
|
14 |
Iterable<ProjectTsv> findByFundingpathidStartingWith(String fundingpathid); |
|
15 |
|
|
16 |
} |
modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/project/JdbcApiDao.java | ||
---|---|---|
1 | 1 |
package eu.dnetlib.openaire.exporter.project; |
2 | 2 |
|
3 |
import java.io.BufferedOutputStream; |
|
3 | 4 |
import java.io.IOException; |
4 | 5 |
import java.io.OutputStream; |
5 | 6 |
import java.nio.charset.Charset; |
6 |
import java.sql.*; |
|
7 |
import java.sql.Connection; |
|
8 |
import java.sql.PreparedStatement; |
|
9 |
import java.sql.ResultSet; |
|
10 |
import java.sql.SQLException; |
|
7 | 11 |
import java.time.Duration; |
8 | 12 |
import java.time.LocalDateTime; |
9 |
import java.util.Arrays; |
|
10 | 13 |
import java.util.List; |
14 |
import java.util.concurrent.ScheduledThreadPoolExecutor; |
|
15 |
import java.util.concurrent.atomic.AtomicInteger; |
|
16 |
import java.util.zip.GZIPOutputStream; |
|
17 |
import java.util.zip.ZipEntry; |
|
11 | 18 |
import java.util.zip.ZipOutputStream; |
12 | 19 |
|
13 |
import com.google.common.base.Function; |
|
20 |
import javax.annotation.PostConstruct; |
|
21 |
|
|
14 | 22 |
import com.google.common.base.Joiner; |
15 | 23 |
import com.google.common.base.Splitter; |
16 |
import com.google.common.collect.Iterables; |
|
17 |
import com.google.common.collect.Lists; |
|
24 |
import com.google.common.util.concurrent.ListeningExecutorService; |
|
25 |
import com.google.common.util.concurrent.MoreExecutors; |
|
26 |
import com.google.common.util.concurrent.ThreadFactoryBuilder; |
|
27 |
import eu.dnetlib.OpenaireExporterConfig; |
|
28 |
import eu.dnetlib.openaire.exporter.datasource.ApiException; |
|
18 | 29 |
import eu.dnetlib.openaire.exporter.model.project.Project; |
30 |
import eu.dnetlib.openaire.exporter.model.project.ProjectTsv; |
|
31 |
import eu.dnetlib.openaire.exporter.project.repository.ProjectDetailsRepository; |
|
32 |
import eu.dnetlib.openaire.exporter.project.repository.ProjectTsvRepository; |
|
19 | 33 |
import org.antlr.stringtemplate.StringTemplate; |
20 | 34 |
import org.apache.commons.dbcp2.BasicDataSource; |
21 | 35 |
import org.apache.commons.lang3.StringUtils; |
22 | 36 |
import org.apache.commons.logging.Log; |
23 | 37 |
import org.apache.commons.logging.LogFactory; |
24 | 38 |
import org.springframework.beans.factory.annotation.Autowired; |
25 |
import org.springframework.beans.factory.annotation.Value; |
|
26 | 39 |
import org.springframework.stereotype.Component; |
27 | 40 |
|
28 | 41 |
/** |
... | ... | |
32 | 45 |
public class JdbcApiDao { |
33 | 46 |
|
34 | 47 |
public static final Charset UTF8 = Charset.forName("UTF-8"); |
48 |
|
|
35 | 49 |
private static final Log log = LogFactory.getLog(JdbcApiDao.class); |
36 |
@Value("${openaire.exporter.jdbc.maxrows}") |
|
37 |
private int maxRows; |
|
38 | 50 |
|
39 |
@Value("${openaire.project.exporter.projectdetails.flushsize}")
|
|
40 |
private int gzipFlushSize;
|
|
51 |
@Autowired
|
|
52 |
private OpenaireExporterConfig config;
|
|
41 | 53 |
|
42 |
@Value("${openaire.project.exporter.projects2tsv.fields}") |
|
43 |
private String tsvFields; |
|
44 |
|
|
45 | 54 |
@Autowired |
46 | 55 |
private BasicDataSource projectApiDataSource; |
47 | 56 |
|
48 |
public void streamProjects(final String sql, final OutputStream out, |
|
49 |
final String head, final StringTemplate projectTemplate, final String tail, |
|
50 |
final ValueCleaner cleaner) throws IOException, SQLException { |
|
57 |
@Autowired |
|
58 |
private ProjectDetailsRepository projectDetailsRepository; |
|
51 | 59 |
|
52 |
if (log.isDebugEnabled()) { |
|
53 |
log.debug("Thread " + Thread.currentThread().getId() + " begin"); |
|
54 |
} |
|
55 |
final LocalDateTime start = LocalDateTime.now(); |
|
60 |
@Autowired |
|
61 |
private ProjectTsvRepository projectTsvRepository; |
|
56 | 62 |
|
57 |
if (StringUtils.isNotBlank(head)) { |
|
58 |
out.write(head.getBytes(UTF8)); |
|
59 |
} |
|
63 |
private ListeningExecutorService service; |
|
60 | 64 |
|
61 |
try(final Connection con = getConn(); final PreparedStatement stm = getStm(sql, con); final ResultSet rs = stm.executeQuery()) { |
|
62 |
while(rs.next()){ |
|
63 |
final Project p = new Project() |
|
64 |
.setFunder(cleaner.clean(rs.getString("funder"))) |
|
65 |
.setJurisdiction(cleaner.clean(rs.getString("jurisdiction"))) |
|
66 |
.setFundingpathid(cleaner.clean(rs.getString("fundingpathid"))) |
|
67 |
.setAcronym(cleaner.clean(rs.getString("acronym"))) |
|
68 |
.setTitle(cleaner.clean(rs.getString("title"))) |
|
69 |
.setCode(cleaner.clean(rs.getString("code"))) |
|
70 |
.setStartdate(cleaner.clean(rs.getString("startdate"))) |
|
71 |
.setEnddate(cleaner.clean(rs.getString("enddate"))); |
|
65 |
@PostConstruct |
|
66 |
public void init() { |
|
67 |
service = MoreExecutors.listeningDecorator( |
|
68 |
new ScheduledThreadPoolExecutor(config.getRequestWorkers(), |
|
69 |
new ThreadFactoryBuilder().setNameFormat("project-exporter-dao-%d").build())); |
|
70 |
} |
|
72 | 71 |
|
73 |
projectTemplate.reset(); |
|
74 |
projectTemplate.setAttribute("p", p); |
|
75 |
out.write(projectTemplate.toString().getBytes(UTF8)); |
|
72 |
public void processProjectDetails(final OutputStream outputStream, String format, Boolean compress) throws IOException { |
|
73 |
service.execute(() -> { |
|
74 |
try { |
|
75 |
final OutputStream out = getOutputStream(outputStream, compress); |
|
76 |
try { |
|
77 |
final AtomicInteger i = new AtomicInteger(0); |
|
78 |
projectDetailsRepository.findAll().forEach(p -> { |
|
79 |
try { |
|
80 |
switch (format) { |
|
81 |
case "csv": |
|
82 |
out.write(p.asCSV().getBytes(UTF8)); |
|
83 |
break; |
|
84 |
case "json": |
|
85 |
out.write(p.asJson().getBytes(UTF8)); |
|
86 |
break; |
|
87 |
} |
|
88 |
i.incrementAndGet(); |
|
89 |
if (i.intValue() % config.getProject().getGzipFlushSize() == 0) { |
|
90 |
log.debug("flushing output stream"); |
|
91 |
out.flush(); |
|
92 |
} |
|
93 |
} catch (IOException e) { |
|
94 |
throw new RuntimeException(e); |
|
95 |
} |
|
96 |
} |
|
97 |
); |
|
98 |
} finally { |
|
99 |
if (out instanceof GZIPOutputStream) { |
|
100 |
((GZIPOutputStream) out).finish(); |
|
101 |
} |
|
102 |
out.close(); |
|
103 |
} |
|
104 |
} catch (IOException e) { |
|
105 |
throw new RuntimeException(e); |
|
76 | 106 |
} |
77 |
if (StringUtils.isNotBlank(tail)) { |
|
78 |
out.write(tail.getBytes(UTF8)); |
|
79 |
} |
|
80 |
final LocalDateTime end = LocalDateTime.now(); |
|
81 |
if (log.isDebugEnabled()) { |
|
82 |
log.debug("Thread " + Thread.currentThread().getId() + " ends, took: " + Duration.between(start, end)); |
|
83 |
} |
|
84 |
} finally { |
|
85 |
out.close(); |
|
86 |
} |
|
107 |
}); |
|
87 | 108 |
} |
88 | 109 |
|
89 |
public void streamProjectsTSV(final String sql, final ZipOutputStream out) throws IOException, SQLException { |
|
90 |
|
|
91 |
if (log.isDebugEnabled()) { |
|
92 |
log.debug("Thread " + Thread.currentThread().getId() + " begin"); |
|
110 |
private OutputStream getOutputStream(final OutputStream outputStream, final Boolean compress) throws IOException { |
|
111 |
if (compress != null && compress) { |
|
112 |
return new GZIPOutputStream(outputStream); |
|
93 | 113 |
} |
94 |
final LocalDateTime start = LocalDateTime.now(); |
|
95 |
final List<String> fields = Lists.newArrayList(Splitter.on(",").omitEmptyStrings().trimResults().split(tsvFields)); |
|
96 |
writeCSVLine(out, fields); |
|
97 |
try(final Connection con = getConn(); final PreparedStatement stm = getStm(sql, con); final ResultSet rs = stm.executeQuery()) { |
|
98 |
while(rs.next()) { |
|
99 |
final Project p = new Project() |
|
100 |
.setCode(rs.getString("code")) |
|
101 |
.setAcronym(rs.getString("acronym")) |
|
102 |
.setTitle(rs.getString("title")) |
|
103 |
.setCallIdentifier(rs.getString("call_identifier")) |
|
104 |
.setStartdate(rs.getString("startdate")) |
|
105 |
.setEnddate(rs.getString("enddate")) |
|
106 |
.setOaMandateForPublications(rs.getBoolean("oa_mandate_for_publications")) |
|
107 |
.setOaMandateForDatasets(rs.getBoolean("oa_mandate_for_datasets")) |
|
108 |
.setDescription(rs.getString("description")) |
|
109 |
.setOrgLegalname(rs.getString("legalname")) |
|
110 |
.setOrgCountry(rs.getString("country")) |
|
111 |
.setOrgRole(rs.getString("role")) |
|
112 |
.setFirstname(rs.getString("firstname")) |
|
113 |
.setSecondnames(rs.getString("secondnames")) |
|
114 |
.setEmail(rs.getString("email")); |
|
114 |
return outputStream; |
|
115 |
} |
|
115 | 116 |
|
116 |
writeCSVLine(out, p.asList()); |
|
117 |
public void processTsvRequest(final OutputStream outputStream, final Boolean article293, final String fundingPrefix, final String filename) throws ApiException { |
|
118 |
service.execute(() -> { |
|
119 |
try(final ZipOutputStream out = new ZipOutputStream(new BufferedOutputStream(outputStream))) { |
|
120 |
out.putNextEntry(new ZipEntry(filename)); |
|
121 |
writeTsvLine(out, Splitter.on(",").trimResults().splitToList(config.getProject().getTsvFields())); |
|
122 |
queryForTsv(fundingPrefix, article293).forEach(p -> { |
|
123 |
try { |
|
124 |
writeTsvLine(out, p.asList()); |
|
125 |
} catch (IOException e) { |
|
126 |
throw new RuntimeException(e); |
|
127 |
} |
|
128 |
}); |
|
129 |
} catch (Throwable e) { |
|
130 |
throw new RuntimeException("Error processing the request", e); |
|
117 | 131 |
} |
118 |
out.closeEntry(); |
|
119 |
final LocalDateTime end = LocalDateTime.now(); |
|
120 |
if (log.isDebugEnabled()) { |
|
121 |
log.debug("Thread " + Thread.currentThread().getId() + " ends, took: " + Duration.between(start, end)); |
|
122 |
} |
|
123 |
} finally { |
|
124 |
out.close(); |
|
125 |
} |
|
132 |
}); |
|
126 | 133 |
} |
127 | 134 |
|
128 |
private void writeCSVLine(final ZipOutputStream out, final List<String> list) throws IOException {
|
|
129 |
out.write(Joiner.on('\t').useForNull("").join(list).getBytes(UTF8));
|
|
135 |
private void writeTsvLine(final ZipOutputStream out, final List<String> s) throws IOException {
|
|
136 |
out.write(Joiner.on('\t').useForNull("").join(s).getBytes(UTF8));
|
|
130 | 137 |
out.write('\n'); |
131 | 138 |
} |
132 | 139 |
|
140 |
private Iterable<ProjectTsv> queryForTsv(final String fundingPrefix, final Boolean article293) { |
|
141 |
log.debug(String.format("fundingPrefix:'%s' and article293:'%s'", fundingPrefix, article293)); |
|
142 |
if (article293 != null) { |
|
143 |
return projectTsvRepository.findByFundingpathidStartingWithAndOaMandateForDatasetsOrderByAcronym(fundingPrefix, article293); |
|
144 |
} else { |
|
145 |
return projectTsvRepository.findByFundingpathidStartingWithOrderByAcronym(fundingPrefix); |
|
146 |
} |
|
147 |
} |
|
133 | 148 |
|
149 |
public void streamProjects(final String sql, final OutputStream out, |
|
150 |
final String head, final StringTemplate projectTemplate, final String tail, |
|
151 |
final ValueCleaner cleaner) throws IOException, SQLException { |
|
152 |
service.execute(() -> { |
|
153 |
try { |
|
154 |
|
|
155 |
if (log.isDebugEnabled()) { |
|
156 |
log.debug("Thread " + Thread.currentThread().getId() + " begin"); |
|
157 |
} |
|
158 |
final LocalDateTime start = LocalDateTime.now(); |
|
159 |
|
|
160 |
if (StringUtils.isNotBlank(head)) { |
|
161 |
out.write(head.getBytes(UTF8)); |
|
162 |
} |
|
163 |
|
|
164 |
try (final Connection con = getConn(); final PreparedStatement stm = getStm(sql, con); final ResultSet rs = stm.executeQuery()) { |
|
165 |
while (rs.next()) { |
|
166 |
final Project p = new Project() |
|
167 |
.setFunder(cleaner.clean(rs.getString("funder"))) |
|
168 |
.setJurisdiction(cleaner.clean(rs.getString("jurisdiction"))) |
|
169 |
.setFundingpathid(cleaner.clean(rs.getString("fundingpathid"))) |
|
170 |
.setAcronym(cleaner.clean(rs.getString("acronym"))) |
|
171 |
.setTitle(cleaner.clean(rs.getString("title"))) |
|
172 |
.setCode(cleaner.clean(rs.getString("code"))) |
|
173 |
.setStartdate(cleaner.clean(rs.getString("startdate"))) |
|
174 |
.setEnddate(cleaner.clean(rs.getString("enddate"))); |
|
175 |
|
|
176 |
projectTemplate.reset(); |
|
177 |
projectTemplate.setAttribute("p", p); |
|
178 |
out.write(projectTemplate.toString().getBytes(UTF8)); |
|
179 |
} |
|
180 |
if (StringUtils.isNotBlank(tail)) { |
|
181 |
out.write(tail.getBytes(UTF8)); |
|
182 |
} |
|
183 |
final LocalDateTime end = LocalDateTime.now(); |
|
184 |
if (log.isDebugEnabled()) { |
|
185 |
log.debug("Thread " + Thread.currentThread().getId() + " ends, took: " + Duration.between(start, end)); |
|
186 |
} |
|
187 |
} finally { |
|
188 |
out.close(); |
|
189 |
} |
|
190 |
} catch (IOException | SQLException e) { |
|
191 |
log.error(e); |
|
192 |
throw new RuntimeException(e); |
|
193 |
} |
|
194 |
}); |
|
195 |
} |
|
196 |
|
|
134 | 197 |
private Connection getConn() throws SQLException { |
135 | 198 |
final Connection conn = projectApiDataSource.getConnection(); |
136 | 199 |
conn.setAutoCommit(false); |
... | ... | |
139 | 202 |
|
140 | 203 |
private PreparedStatement getStm(final String sql, final Connection con) throws SQLException { |
141 | 204 |
final PreparedStatement stm = con.prepareStatement(sql); |
142 |
stm.setFetchSize(maxRows);
|
|
205 |
stm.setFetchSize(config.getJdbc().getMaxRows());
|
|
143 | 206 |
return stm; |
144 | 207 |
} |
145 | 208 |
|
146 |
private List<String> asList(final Array value) throws SQLException { |
|
147 |
if (value != null) { |
|
148 |
final List<Object> list = Arrays.asList((Object[]) value.getArray()); |
|
149 |
return Lists.newArrayList(Iterables.transform(list, new Function<Object, String>() { |
|
150 |
@Override |
|
151 |
public String apply(final Object o) { |
|
152 |
return o != null ? o.toString() : null; |
|
153 |
|
|
154 |
} |
|
155 |
})); |
|
156 |
} |
|
157 |
return Lists.newArrayList(); |
|
158 |
} |
|
159 |
|
|
160 | 209 |
} |
modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/project/ProjectQueryParamsFactory.java | ||
---|---|---|
17 | 17 |
final String endUntil) { |
18 | 18 |
ProjectQueryParams params = new ProjectQueryParams(); |
19 | 19 |
|
20 |
String[] arr = request.getPathInfo().replace(BASE_PATH, "").split("\\/");
|
|
20 |
String[] arr = request.getServletPath().replace(BASE_PATH, "").split("\\/");
|
|
21 | 21 |
if (arr.length != 5) throw new IllegalArgumentException("Invalid url"); |
22 | 22 |
|
23 | 23 |
params.setFundingProgramme(arr[0]); |
modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/project/ProjectsController.java | ||
---|---|---|
2 | 2 |
|
3 | 3 |
import java.io.BufferedOutputStream; |
4 | 4 |
import java.io.IOException; |
5 |
import java.io.OutputStream; |
|
6 | 5 |
import java.sql.SQLException; |
7 | 6 |
import java.text.SimpleDateFormat; |
8 | 7 |
import java.util.Arrays; |
9 | 8 |
import java.util.Date; |
10 |
import java.util.Map; |
|
11 |
import java.util.concurrent.atomic.AtomicInteger; |
|
12 |
import java.util.zip.GZIPOutputStream; |
|
13 |
import java.util.zip.ZipEntry; |
|
14 |
import java.util.zip.ZipOutputStream; |
|
15 |
import javax.servlet.ServletOutputStream; |
|
16 | 9 |
import javax.servlet.ServletResponse; |
17 | 10 |
import javax.servlet.http.HttpServletRequest; |
18 | 11 |
import javax.servlet.http.HttpServletResponse; |
19 | 12 |
|
20 |
import com.google.common.base.Joiner; |
|
21 |
import com.google.common.collect.Maps; |
|
22 | 13 |
import com.google.common.xml.XmlEscapers; |
14 |
import eu.dnetlib.OpenaireExporterConfig; |
|
15 |
import eu.dnetlib.OpenaireExporterConfig.Project; |
|
23 | 16 |
import eu.dnetlib.common.rmi.DNetRestDocumentation; |
24 |
import eu.dnetlib.openaire.exporter.datasource.ApiException; |
|
25 |
import eu.dnetlib.openaire.exporter.model.project.ProjectTsv; |
|
26 |
import eu.dnetlib.openaire.exporter.project.repository.ProjectDetailsRepository; |
|
27 |
import eu.dnetlib.openaire.exporter.project.repository.ProjectTsvRepository; |
|
28 | 17 |
import org.antlr.stringtemplate.StringTemplate; |
29 | 18 |
import org.apache.commons.io.IOUtils; |
30 | 19 |
import org.apache.commons.lang3.StringUtils; |
... | ... | |
32 | 21 |
import org.apache.commons.logging.Log; |
33 | 22 |
import org.apache.commons.logging.LogFactory; |
34 | 23 |
import org.springframework.beans.factory.annotation.Autowired; |
35 |
import org.springframework.beans.factory.annotation.Value; |
|
36 | 24 |
import org.springframework.core.io.Resource; |
37 | 25 |
import org.springframework.http.HttpStatus; |
38 | 26 |
import org.springframework.stereotype.Controller; |
... | ... | |
46 | 34 |
|
47 | 35 |
public final static String UTF8 = "UTF-8"; |
48 | 36 |
|
49 |
@Value("${openaire.project.exporter.projectsfundings.sql.template}")
|
|
50 |
private Resource projectsFundingQueryTemplate;
|
|
37 |
@Autowired
|
|
38 |
private OpenaireExporterConfig config;
|
|
51 | 39 |
|
52 |
@Value("${openaire.project.exporter.projects2tsv.sql.template}") |
|
53 |
private Resource projects2tsvQueryTemplate; |
|
54 |
|
|
55 |
@Value("${openaire.project.exporter.dspace.template.project}") |
|
56 |
private Resource dspaceTemplate; |
|
57 |
|
|
58 |
@Value("${openaire.project.exporter.dspace.template.head}") |
|
59 |
private Resource dspaceHeadTemplate; |
|
60 |
|
|
61 |
@Value("${openaire.project.exporter.dspace.template.tail}") |
|
62 |
private Resource dspaceTailTemplate; |
|
63 |
|
|
64 |
@Value("${openaire.project.exporter.eprints.template}") |
|
65 |
private Resource eprintsTemplate; |
|
66 |
|
|
67 |
@Value("${openaire.project.exporter.projectdetails.sql}") |
|
68 |
private Resource projectDetailsSql; |
|
69 |
|
|
70 |
@Value("${openaire.project.exporter.projectdetails.flushsize}") |
|
71 |
private int gzipFlushSize; |
|
72 |
|
|
73 | 40 |
@Autowired |
74 | 41 |
private JdbcApiDao dao; |
75 | 42 |
|
76 | 43 |
@Autowired |
77 | 44 |
private ProjectQueryParamsFactory projectQueryParamsFactory; |
78 | 45 |
|
79 |
@Autowired |
|
80 |
private ProjectDetailsRepository projectDetailsRepository; |
|
81 |
|
|
82 |
@Autowired |
|
83 |
private ProjectTsvRepository projectTsvRepository; |
|
84 |
|
|
85 | 46 |
@Deprecated |
86 | 47 |
/** |
87 | 48 |
* @deprecated we should use the codes returned by the yet to come service |
... | ... | |
91 | 52 |
} |
92 | 53 |
|
93 | 54 |
@RequestMapping(value = "/openaire/export/**/project/dspace.do", method = RequestMethod.GET) |
94 |
void processDspace(final HttpServletRequest request, |
|
95 |
final ServletResponse response, |
|
55 |
void processDspace(final HttpServletRequest request, final ServletResponse response, |
|
96 | 56 |
@RequestParam(value = "startFrom", required = false) final String startFrom, |
97 | 57 |
@RequestParam(value = "startUntil", required = false) final String startUntil, |
98 | 58 |
@RequestParam(value = "endFrom", required = false) final String endFrom, |
99 | 59 |
@RequestParam(value = "endUntil", required = false) final String endUntil) throws Exception { |
100 | 60 |
|
61 |
final Project conf = config.getProject(); |
|
101 | 62 |
|
102 | 63 |
final ProjectQueryParams params = projectQueryParamsFactory.generateParams(request, startFrom, startUntil, endFrom, endUntil); |
103 |
final StringTemplate headSt = new StringTemplate(IOUtils.toString(dspaceHeadTemplate.getInputStream(), UTF8 ));
|
|
64 |
final StringTemplate headSt = new StringTemplate(IOUtils.toString(conf.getDspaceHeadTemplate().getInputStream(), UTF8));
|
|
104 | 65 |
|
105 | 66 |
headSt.setAttribute("fundingProgramme", params.getFundingProgramme()); |
106 | 67 |
|
107 |
final StringTemplate tailSt = new StringTemplate(IOUtils.toString(dspaceTailTemplate.getInputStream(), UTF8));
|
|
68 |
final StringTemplate tailSt = new StringTemplate(IOUtils.toString(conf.getDspaceTailTemplate().getInputStream(), UTF8));
|
|
108 | 69 |
|
109 | 70 |
response.setContentType("text/xml"); |
110 |
doProcess(response, params, headSt.toString(), dspaceTemplate, tailSt.toString(), s -> XmlEscapers.xmlContentEscaper().escape(oneLiner(s)));
|
|
71 |
doProcess(response, params, headSt.toString(), conf.getDspaceTemplate(), tailSt.toString(), s -> XmlEscapers.xmlContentEscaper().escape(oneLiner(s)));
|
|
111 | 72 |
} |
112 | 73 |
|
113 | 74 |
@RequestMapping(value = "/openaire/export/**/project/eprints.do", method = RequestMethod.GET) |
114 |
void processEprints(final HttpServletRequest request, |
|
115 |
final ServletResponse response, |
|
75 |
void processEprints(final HttpServletRequest request, final ServletResponse response, |
|
116 | 76 |
@RequestParam(value = "startFrom", required = false) final String startFrom, |
117 | 77 |
@RequestParam(value = "startUntil", required = false) final String startUntil, |
118 | 78 |
@RequestParam(value = "endFrom", required = false) final String endFrom, |
119 | 79 |
@RequestParam(value = "endUntil", required = false) final String endUntil) throws Exception { |
120 | 80 |
|
81 |
final Project conf = config.getProject(); |
|
121 | 82 |
final ProjectQueryParams params = projectQueryParamsFactory.generateParams(request, startFrom, startUntil, endFrom, endUntil); |
122 | 83 |
response.setContentType("text/html"); |
123 |
doProcess(response, params, null, eprintsTemplate, null, s -> oneLiner(s));
|
|
84 |
doProcess(response, params, null, conf.getEprintsTemplate(), null, s -> oneLiner(s));
|
|
124 | 85 |
} |
125 | 86 |
|
126 | 87 |
private String oneLiner(final String s) { |
... | ... | |
134 | 95 |
final ValueCleaner cleaner) throws IOException, SQLException { |
135 | 96 |
|
136 | 97 |
final StringTemplate st = new StringTemplate(IOUtils.toString(projectTemplate.getInputStream(), UTF8)); |
137 |
dao.streamProjects(obtainQuery(params), response.getOutputStream(), head, st, tail, cleaner);
|
|
98 |
dao.streamProjects(obtainQuery(params), new BufferedOutputStream(response.getOutputStream()), head, st, tail, cleaner);
|
|
138 | 99 |
} |
139 | 100 |
|
140 | 101 |
@RequestMapping(value = "/openaire/export/project2tsv.do", method = RequestMethod.GET) |
... | ... | |
142 | 103 |
@RequestParam(value = "funding", required = true) final String funding, |
143 | 104 |
@RequestParam(value = "article293", required = false) final Boolean article293) throws Exception { |
144 | 105 |
|
145 |
final String date = new SimpleDateFormat("yyyyMMdd").format(new Date()); |
|
146 |
final String filename = "projects_" + funding + "_" + date + ".tsv"; |
|
147 |
response.setContentType("text/tab-separated-values"); |
|
148 |
response.setHeader("Content-Disposition", "attachment; filename=\"" + filename + ".zip\""); |
|
149 |
|
|
150 |
final StringTemplate st = new StringTemplate(IOUtils.toString(projects2tsvQueryTemplate.getInputStream(), UTF8)); |
|
151 |
Funding requestedFunding = Funding.valueOf(funding.toUpperCase()); |
|
152 |
String fundingPrefix = getFundingPrefix(requestedFunding, null); |
|
153 |
log.debug("Setting fundingprefix to " + fundingPrefix); |
|
154 |
st.setAttribute("fundingprefix", fundingPrefix); |
|
155 |
st.setAttribute("filters", expandFilters(article293)); |
|
156 |
|
|
157 |
final ZipOutputStream out = new ZipOutputStream(new BufferedOutputStream(response.getOutputStream())); |
|
158 |
out.putNextEntry(new ZipEntry(filename)); |
|
159 |
dao.streamProjectsTSV(st.toString(), out); |
|
160 |
} |
|
161 |
|
|
162 |
@RequestMapping(value = "/openaire/export/project2tsv2.do", method = RequestMethod.GET) |
|
163 |
void processTsv2(final HttpServletRequest request, final HttpServletResponse response, |
|
164 |
@RequestParam(value = "funding", required = true) final String funding, |
|
165 |
@RequestParam(value = "article293", required = false) final Boolean article293) throws Exception { |
|
166 |
|
|
167 | 106 |
final Funding requestedFunding = Funding.valueOf(funding.toUpperCase()); |
168 | 107 |
final String fundingPrefix = getFundingPrefix(requestedFunding, null); |
169 | 108 |
|
... | ... | |
172 | 111 |
response.setContentType("text/tab-separated-values"); |
173 | 112 |
response.setHeader("Content-Disposition", "attachment; filename=\"" + filename + ".zip\""); |
174 | 113 |
|
175 |
try(final ZipOutputStream out = new ZipOutputStream(new BufferedOutputStream(response.getOutputStream()))) { |
|
176 |
out.putNextEntry(new ZipEntry(filename)); |
|
177 |
queryForTsv(fundingPrefix, article293).forEach(p -> { |
|
178 |
try { |
|
179 |
out.write(Joiner.on('\t').useForNull("").join(p.asList()).getBytes(UTF8)); |
|
180 |
out.write('\n'); |
|
181 |
} catch (IOException e) { |
|
182 |
throw new RuntimeException(e); |
|
183 |
} |
|
184 |
}); |
|
185 |
} catch (Throwable e) { |
|
186 |
throw new ApiException(org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error processing the request", e); |
|
187 |
} |
|
114 |
dao.processTsvRequest(response.getOutputStream(), article293, fundingPrefix, filename); |
|
188 | 115 |
} |
189 | 116 |
|
190 |
private Iterable<ProjectTsv> queryForTsv(final String fundingPrefix, final Boolean article293) { |
|
191 |
log.debug(String.format("fundingPrefix:'%s' and article293:'%s'", fundingPrefix, article293)); |
|
192 |
if (article293 != null) { |
|
193 |
return projectTsvRepository.findByFundingpathidStartingWithAndOaMandateForDatasetsOrderByAcronym(fundingPrefix, article293); |
|
194 |
} else { |
|
195 |
return projectTsvRepository.findByFundingpathidStartingWithOrderByAcronym(fundingPrefix); |
|
196 |
} |
|
197 |
} |
|
198 |
|
|
199 | 117 |
@RequestMapping(value = "/openaire/export/streamProjectDetails.do", method = RequestMethod.GET) |
200 | 118 |
void streamProjectDetails(final HttpServletResponse response, |
201 | 119 |
@RequestParam(value = "format", required = true) final String format, |
... | ... | |
214 | 132 |
default: throw new IllegalArgumentException("unsupported format: " + format); |
215 | 133 |
} |
216 | 134 |
|
217 |
final OutputStream out = getOutputStream(response.getOutputStream(), compress); |
|
218 |
try { |
|
219 |
final AtomicInteger i = new AtomicInteger(0); |
|
220 |
projectDetailsRepository.findAll().forEach(p -> { |
|
221 |
try { |
|
222 |
switch (format) { |
|
223 |
case "csv": |
|
224 |
out.write(p.asCSV().getBytes(UTF8)); |
|
225 |
break; |
|
226 |
case "json": |
|
227 |
out.write(p.asJson().getBytes(UTF8)); |
|
228 |
break; |
|
229 |
} |
|
230 |
i.incrementAndGet(); |
|
231 |
if (i.intValue() % gzipFlushSize == 0) { |
|
232 |
log.debug("flushing output stream"); |
|
233 |
out.flush(); |
|
234 |
} |
|
235 |
} catch (IOException e) { |
|
236 |
throw new RuntimeException(e); |
|
237 |
} |
|
238 |
} |
|
239 |
); |
|
240 |
} finally { |
|
241 |
if (out instanceof GZIPOutputStream) { |
|
242 |
((GZIPOutputStream) out).finish(); |
|
243 |
} |
|
244 |
out.close(); |
|
245 |
} |
|
135 |
dao.processProjectDetails(response.getOutputStream(), format, compress); |
|
246 | 136 |
} |
247 | 137 |
|
248 |
private OutputStream getOutputStream(final ServletOutputStream outputStream, final Boolean compress) throws IOException { |
|
249 |
if (compress != null && compress) { |
|
250 |
return new GZIPOutputStream(outputStream); |
|
251 |
} |
|
252 |
return outputStream; |
|
253 |
} |
|
254 |
|
|
255 | 138 |
@ExceptionHandler({Exception.class, Throwable.class}) |
256 | 139 |
@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) |
257 | 140 |
public @ResponseBody ErrorMessage handleSqlException(final Exception e) { |
... | ... | |
286 | 169 |
} |
287 | 170 |
} |
288 | 171 |
|
289 |
|
|
290 |
private Map<String, String> expandFilters(final Boolean article293) { |
|
291 |
final Map<String, String> res = Maps.newHashMap(); |
|
292 |
|
|
293 |
if (article293 != null) { |
|
294 |
res.put("ec_article29_3", String.valueOf(article293)); |
|
295 |
} |
|
296 |
|
|
297 |
return res; |
|
298 |
} |
|
299 |
|
|
300 | 172 |
/** |
301 | 173 |
* Creates the query on the fundingProgramme specified in the given parameters. |
302 | 174 |
* |
... | ... | |
314 | 186 |
String funding = params.getFundingProgramme(); |
315 | 187 |
String suffix = params.getFundingPath(); |
316 | 188 |
String fundingPrefix = getFundingPrefix(Funding.valueOf(funding.toUpperCase()), suffix); |
317 |
final StringTemplate st = new StringTemplate(IOUtils.toString(projectsFundingQueryTemplate.getInputStream(), UTF8));
|
|
189 |
final StringTemplate st = new StringTemplate(IOUtils.toString(config.getProject().getProjectsFundingQueryTemplate().getInputStream(), UTF8));
|
|
318 | 190 |
st.setAttribute("fundingprefix", fundingPrefix); |
319 | 191 |
String theQuery = setDateParameters(st.toString(), params); |
320 | 192 |
log.debug("Generated query: " + theQuery); |
... | ... | |
391 | 263 |
return queryWithDates; |
392 | 264 |
} |
393 | 265 |
|
394 |
public void setProjectsFundingQueryTemplate(final Resource projectsFundingQueryTemplate) { |
|
395 |
this.projectsFundingQueryTemplate = projectsFundingQueryTemplate; |
|
396 |
} |
|
397 |
|
|
398 | 266 |
} |
modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/datasource/clients/ISLookupClient.java | ||
---|---|---|
1 | 1 |
package eu.dnetlib.openaire.exporter.datasource.clients; |
2 | 2 |
|
3 |
import java.nio.charset.Charset; |
|
4 |
|
|
5 |
import eu.dnetlib.OpenaireExporterConfig; |
|
3 | 6 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; |
4 | 7 |
import eu.dnetlib.openaire.exporter.datasource.ApiException; |
5 | 8 |
import eu.dnetlib.openaire.exporter.datasource.clients.utils.IndexDsInfo; |
... | ... | |
7 | 10 |
import org.apache.commons.logging.Log; |
8 | 11 |
import org.apache.commons.logging.LogFactory; |
9 | 12 |
import org.springframework.beans.factory.annotation.Autowired; |
10 |
import org.springframework.beans.factory.annotation.Value; |
|
11 | 13 |
import org.springframework.cache.annotation.Cacheable; |
12 |
import org.springframework.core.io.ClassPathResource; |
|
13 | 14 |
import org.springframework.http.HttpStatus; |
14 | 15 |
import org.springframework.stereotype.Component; |
15 | 16 |
|
... | ... | |
22 | 23 |
private static final Log log = LogFactory.getLog(ISLookupClient.class); |
23 | 24 |
|
24 | 25 |
@Autowired |
26 |
private OpenaireExporterConfig config; |
|
27 |
|
|
28 |
@Autowired |
|
25 | 29 |
private ISLookUpService isLookUpService; |
26 | 30 |
|
27 |
@Value(value = "${openaire.datasource.publisher.findSolrIndexUrl.xquery}") |
|
28 |
private ClassPathResource findSolrIndexUrl; |
|
29 |
|
|
30 |
@Value(value = "${openaire.datasource.publisher.findIndexDsInfo.xquery}") |
|
31 |
private ClassPathResource findIndexDsInfo; |
|
32 |
|
|
33 | 31 |
@Cacheable("datasources-is-cache") |
34 | 32 |
public IndexDsInfo calculateCurrentIndexDsInfo() throws ApiException { |
35 | 33 |
log.warn("calculateCurrentIndexDsInfo(): not using cache"); |
36 | 34 |
try { |
37 |
final String queryUrl = IOUtils.toString(findSolrIndexUrl.getInputStream());
|
|
38 |
final String queryDs = IOUtils.toString(findIndexDsInfo.getInputStream());
|
|
35 |
final String queryUrl = IOUtils.toString(config.getFindSolrIndexUrl().getInputStream(), Charset.defaultCharset());
|
|
36 |
final String queryDs = IOUtils.toString(config.getFindIndexDsInfo().getInputStream(), Charset.defaultCharset());
|
|
39 | 37 |
|
40 | 38 |
final String indexBaseUrl = isLookUpService.getResourceProfileByQuery(queryUrl); |
41 | 39 |
final String[] arr = isLookUpService.getResourceProfileByQuery(queryDs).split("@@@"); |
modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/datasource/clients/DatasourceDao.java | ||
---|---|---|
4 | 4 |
import java.util.Queue; |
5 | 5 |
import java.util.concurrent.*; |
6 | 6 |
|
7 |
import javax.annotation.PostConstruct; |
|
8 |
|
|
7 | 9 |
import com.google.common.collect.Lists; |
8 | 10 |
import com.google.common.util.concurrent.*; |
11 |
import eu.dnetlib.OpenaireExporterConfig; |
|
9 | 12 |
import eu.dnetlib.openaire.exporter.datasource.ApiException; |
10 | 13 |
import eu.dnetlib.openaire.exporter.datasource.clients.utils.IndexDsInfo; |
11 | 14 |
import eu.dnetlib.openaire.exporter.datasource.clients.utils.IndexRecordsInfo; |
... | ... | |
20 | 23 |
import org.apache.commons.logging.LogFactory; |
21 | 24 |
import org.apache.http.HttpStatus; |
22 | 25 |
import org.springframework.beans.factory.annotation.Autowired; |
23 |
import org.springframework.beans.factory.annotation.Value; |
|
24 | 26 |
import org.springframework.data.domain.Pageable; |
25 | 27 |
import org.springframework.data.domain.Slice; |
26 | 28 |
import org.springframework.stereotype.Component; |
... | ... | |
36 | 38 |
private static final Log log = LogFactory.getLog(DatasourceDao.class); |
37 | 39 |
|
38 | 40 |
@Autowired |
41 |
private OpenaireExporterConfig config; |
|
42 |
|
|
43 |
@Autowired |
|
39 | 44 |
private MongoLoggerClient mongoLoggerClient; |
40 | 45 |
|
41 | 46 |
@Autowired |
... | ... | |
70 | 75 |
|
71 | 76 |
private ListeningExecutorService service; |
72 | 77 |
|
73 |
private final static int WORKERS = 100; |
|
74 |
|
|
75 |
@Value("${openaire.datasource.publisher.timeout}") |
|
76 |
private int timeout = 10; |
|
77 |
|
|
78 |
public DatasourceDao() { |
|
78 |
@PostConstruct |
|
79 |
public void init() { |
|
79 | 80 |
service = MoreExecutors.listeningDecorator( |
80 |
new ScheduledThreadPoolExecutor(WORKERS,
|
|
81 |
new ScheduledThreadPoolExecutor(config.getRequestWorkers(),
|
|
81 | 82 |
new ThreadFactoryBuilder().setNameFormat("datasource-info-retriever-%d").build())); |
82 | 83 |
} |
83 | 84 |
|
... | ... | |
99 | 100 |
dsRepository.findByOfficialnameContainingOrEnglishnameContainingAllIgnoreCase(name, name, pageable); |
100 | 101 |
c.addCallback(getSearchCallback(outerLatch, errors, datasourceResponse)); |
Also available in: Unified diff
refactor of the application properties