Revision 48459
Added by Claudio Atzori almost 7 years ago
modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/model/project/ProjectApi.java | ||
---|---|---|
24 | 24 |
|
25 | 25 |
@Id |
26 | 26 |
@JsonIgnore |
27 |
private int id;
|
|
27 |
private String id;
|
|
28 | 28 |
|
29 | 29 |
private String code; |
30 | 30 |
private String acronym; |
... | ... | |
78 | 78 |
return s.replaceAll("/", "%2F"); |
79 | 79 |
} |
80 | 80 |
|
81 |
public int getId() {
|
|
81 |
public String getId() {
|
|
82 | 82 |
return id; |
83 | 83 |
} |
84 | 84 |
|
85 |
public void setId(final int id) {
|
|
85 |
public void setId(final String id) {
|
|
86 | 86 |
this.id = id; |
87 | 87 |
} |
88 | 88 |
|
modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/model/project/ProjectTsv.java | ||
---|---|---|
22 | 22 |
|
23 | 23 |
@Id |
24 | 24 |
@JsonIgnore |
25 |
private int id;
|
|
25 |
private String id;
|
|
26 | 26 |
private String code; |
27 | 27 |
private String acronym; |
28 | 28 |
private String title; |
... | ... | |
74 | 74 |
return StringUtils.isNotBlank(s) ? "\"" + s.replaceAll("\\n|\\t|\\s+", " ").replace("\"","\"\"").trim() + "\"" : ""; |
75 | 75 |
} |
76 | 76 |
|
77 |
public int getId() {
|
|
77 |
public String getId() {
|
|
78 | 78 |
return id; |
79 | 79 |
} |
80 | 80 |
|
81 |
public void setId(final int id) {
|
|
81 |
public void setId(final String id) {
|
|
82 | 82 |
this.id = id; |
83 | 83 |
} |
84 | 84 |
|
modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/project/JdbcApiDao.java | ||
---|---|---|
11 | 11 |
import java.time.Duration; |
12 | 12 |
import java.time.LocalDateTime; |
13 | 13 |
import java.util.List; |
14 |
import java.util.Map; |
|
15 |
import java.util.Set; |
|
14 | 16 |
import java.util.concurrent.atomic.AtomicInteger; |
15 | 17 |
import java.util.zip.GZIPOutputStream; |
16 | 18 |
import java.util.zip.ZipEntry; |
... | ... | |
18 | 20 |
|
19 | 21 |
import com.google.common.base.Joiner; |
20 | 22 |
import com.google.common.base.Splitter; |
23 |
import com.google.common.collect.Maps; |
|
24 |
import com.google.common.collect.Sets; |
|
21 | 25 |
import eu.dnetlib.OpenaireExporterConfig; |
22 | 26 |
import eu.dnetlib.openaire.exporter.datasource.ApiException; |
23 | 27 |
import eu.dnetlib.openaire.exporter.model.project.Project; |
... | ... | |
54 | 58 |
@Autowired |
55 | 59 |
private ProjectTsvRepository projectTsvRepository; |
56 | 60 |
|
61 |
|
|
62 |
public Map<String, String> readFundingpathIds() { |
|
63 |
|
|
64 |
log.debug("loading funding ids"); |
|
65 |
final String sql = "SELECT id FROM fundingpaths"; |
|
66 |
final Set<String> ids = Sets.newHashSet(); |
|
67 |
try (final Connection con = getConn(); final PreparedStatement stm = getStm(sql, con); final ResultSet rs = stm.executeQuery()) { |
|
68 |
while (rs.next()) { |
|
69 |
ids.add(rs.getString("id")); |
|
70 |
} |
|
71 |
} catch (SQLException e) { |
|
72 |
throw new RuntimeException(e); |
|
73 |
} |
|
74 |
log.debug(String.format("loaded %s funding ids", ids.size())); |
|
75 |
|
|
76 |
final Map<String, String> res = Maps.newHashMap(); |
|
77 |
final Splitter sp = Splitter.on("::").trimResults(); |
|
78 |
ids.stream() |
|
79 |
.filter(s -> sp.splitToList(s).size() < 3) |
|
80 |
.forEach(s -> res.put(StringUtils.substringAfterLast(s, "::").toUpperCase(), s)); |
|
81 |
|
|
82 |
res.put("FP7", "ec__________::EC::FP7"); |
|
83 |
res.put("H2020", "ec__________::EC::H2020"); |
|
84 |
log.debug(String.format("processed %s funding ids", res.size())); |
|
85 |
res.forEach((k,v) -> log.debug(String.format("%s : '%s'", k, v))); |
|
86 |
return res; |
|
87 |
} |
|
88 |
|
|
57 | 89 |
public void processProjectDetails(final OutputStream outputStream, String format, Boolean compress) throws IOException { |
90 |
final OutputStream out = getOutputStream(outputStream, compress); |
|
58 | 91 |
try { |
59 |
final OutputStream out = getOutputStream(outputStream, compress); |
|
60 |
try { |
|
61 |
final AtomicInteger i = new AtomicInteger(0); |
|
62 |
projectDetailsRepository.findAll().forEach(p -> { |
|
63 |
try { |
|
64 |
switch (format) { |
|
65 |
case "csv": |
|
66 |
out.write(p.asCSV().getBytes(UTF8)); |
|
67 |
break; |
|
68 |
case "json": |
|
69 |
out.write(p.asJson().getBytes(UTF8)); |
|
70 |
break; |
|
71 |
} |
|
72 |
i.incrementAndGet(); |
|
73 |
if (i.intValue() % config.getProject().getGzipFlushSize() == 0) { |
|
74 |
log.debug("flushing output stream"); |
|
75 |
out.flush(); |
|
76 |
} |
|
77 |
} catch (IOException e) { |
|
78 |
throw new RuntimeException(e); |
|
92 |
final AtomicInteger i = new AtomicInteger(0); |
|
93 |
projectDetailsRepository.findAll().forEach(p -> { |
|
94 |
try { |
|
95 |
switch (format) { |
|
96 |
case "csv": |
|
97 |
out.write(p.asCSV().getBytes(UTF8)); |
|
98 |
break; |
|
99 |
case "json": |
|
100 |
out.write(p.asJson().getBytes(UTF8)); |
|
101 |
break; |
|
79 | 102 |
} |
103 |
i.incrementAndGet(); |
|
104 |
if (i.intValue() % config.getProject().getGzipFlushSize() == 0) { |
|
105 |
log.debug("flushing output stream"); |
|
106 |
out.flush(); |
|
107 |
} |
|
108 |
} catch (IOException e) { |
|
109 |
throw new RuntimeException(e); |
|
80 | 110 |
} |
81 |
); |
|
82 |
} finally { |
|
83 |
if (out instanceof GZIPOutputStream) { |
|
84 |
((GZIPOutputStream) out).finish(); |
|
85 | 111 |
} |
86 |
out.close(); |
|
112 |
); |
|
113 |
} finally { |
|
114 |
if (out instanceof GZIPOutputStream) { |
|
115 |
((GZIPOutputStream) out).finish(); |
|
87 | 116 |
} |
88 |
} catch (IOException e) { |
|
89 |
throw new RuntimeException(e); |
|
117 |
out.close(); |
|
90 | 118 |
} |
91 | 119 |
} |
92 | 120 |
|
... | ... | |
131 | 159 |
final String head, final StringTemplate projectTemplate, final String tail, |
132 | 160 |
final ValueCleaner cleaner) throws IOException, SQLException { |
133 | 161 |
|
134 |
try { |
|
135 |
if (log.isDebugEnabled()) { |
|
136 |
log.debug("Thread " + Thread.currentThread().getId() + " begin"); |
|
137 |
} |
|
138 |
final LocalDateTime start = LocalDateTime.now(); |
|
162 |
if (log.isDebugEnabled()) { |
|
163 |
log.debug("Thread " + Thread.currentThread().getId() + " begin"); |
|
164 |
} |
|
165 |
final LocalDateTime start = LocalDateTime.now(); |
|
139 | 166 |
|
140 |
if (StringUtils.isNotBlank(head)) {
|
|
141 |
out.write(head.getBytes(UTF8));
|
|
142 |
}
|
|
167 |
if (StringUtils.isNotBlank(head)) { |
|
168 |
out.write(head.getBytes(UTF8)); |
|
169 |
} |
|
143 | 170 |
|
144 |
try (final Connection con = getConn(); final PreparedStatement stm = getStm(sql, con); final ResultSet rs = stm.executeQuery()) {
|
|
145 |
while (rs.next()) {
|
|
146 |
final Project p = new Project()
|
|
147 |
.setFunder(cleaner.clean(rs.getString("funder")))
|
|
148 |
.setJurisdiction(cleaner.clean(rs.getString("jurisdiction")))
|
|
149 |
.setFundingpathid(cleaner.clean(rs.getString("fundingpathid")))
|
|
150 |
.setAcronym(cleaner.clean(rs.getString("acronym")))
|
|
151 |
.setTitle(cleaner.clean(rs.getString("title")))
|
|
152 |
.setCode(cleaner.clean(rs.getString("code")))
|
|
153 |
.setStartdate(cleaner.clean(rs.getString("startdate")))
|
|
154 |
.setEnddate(cleaner.clean(rs.getString("enddate")));
|
|
171 |
try (final Connection con = getConn(); final PreparedStatement stm = getStm(sql, con); final ResultSet rs = stm.executeQuery()) { |
|
172 |
while (rs.next()) { |
|
173 |
final Project p = new Project() |
|
174 |
.setFunder(cleaner.clean(rs.getString("funder"))) |
|
175 |
.setJurisdiction(cleaner.clean(rs.getString("jurisdiction"))) |
|
176 |
.setFundingpathid(cleaner.clean(rs.getString("fundingpathid"))) |
|
177 |
.setAcronym(cleaner.clean(rs.getString("acronym"))) |
|
178 |
.setTitle(cleaner.clean(rs.getString("title"))) |
|
179 |
.setCode(cleaner.clean(rs.getString("code"))) |
|
180 |
.setStartdate(cleaner.clean(rs.getString("startdate"))) |
|
181 |
.setEnddate(cleaner.clean(rs.getString("enddate"))); |
|
155 | 182 |
|
156 |
projectTemplate.reset(); |
|
157 |
projectTemplate.setAttribute("p", p); |
|
158 |
out.write(projectTemplate.toString().getBytes(UTF8)); |
|
159 |
} |
|
160 |
if (StringUtils.isNotBlank(tail)) { |
|
161 |
out.write(tail.getBytes(UTF8)); |
|
162 |
} |
|
163 |
final LocalDateTime end = LocalDateTime.now(); |
|
164 |
if (log.isDebugEnabled()) { |
|
165 |
log.debug("Thread " + Thread.currentThread().getId() + " ends, took: " + Duration.between(start, end).toMillis() + " ms"); |
|
166 |
} |
|
167 |
} finally { |
|
168 |
out.close(); |
|
169 |
if (log.isDebugEnabled()) { |
|
170 |
log.debug("Thread " + Thread.currentThread().getId() + " closed output stream"); |
|
171 |
} |
|
183 |
projectTemplate.reset(); |
|
184 |
projectTemplate.setAttribute("p", p); |
|
185 |
out.write(projectTemplate.toString().getBytes(UTF8)); |
|
172 | 186 |
} |
173 |
} catch (Throwable e) { |
|
174 |
log.error(e); |
|
175 |
throw new RuntimeException(e); |
|
187 |
if (StringUtils.isNotBlank(tail)) { |
|
188 |
out.write(tail.getBytes(UTF8)); |
|
189 |
} |
|
190 |
final LocalDateTime end = LocalDateTime.now(); |
|
191 |
if (log.isDebugEnabled()) { |
|
192 |
log.debug("Thread " + Thread.currentThread().getId() + " ends, took: " + Duration.between(start, end).toMillis() + " ms"); |
|
193 |
} |
|
194 |
} finally { |
|
195 |
out.close(); |
|
196 |
if (log.isDebugEnabled()) { |
|
197 |
log.debug("Thread " + Thread.currentThread().getId() + " closed output stream"); |
|
198 |
} |
|
176 | 199 |
} |
177 | 200 |
} |
178 | 201 |
|
modules/dnet-openaire-exporter/trunk/src/main/java/eu/dnetlib/openaire/exporter/project/ProjectsController.java | ||
---|---|---|
4 | 4 |
import java.io.IOException; |
5 | 5 |
import java.sql.SQLException; |
6 | 6 |
import java.text.SimpleDateFormat; |
7 |
import java.util.Arrays; |
|
8 | 7 |
import java.util.Date; |
8 |
import java.util.Map; |
|
9 | 9 |
import javax.servlet.ServletResponse; |
10 | 10 |
import javax.servlet.http.HttpServletRequest; |
11 | 11 |
import javax.servlet.http.HttpServletResponse; |
... | ... | |
43 | 43 |
@Autowired |
44 | 44 |
private ProjectQueryParamsFactory projectQueryParamsFactory; |
45 | 45 |
|
46 |
/** |
|
47 |
* @deprecated we should use the codes returned by the yet to come service |
|
48 |
*/ |
|
49 |
@Deprecated |
|
50 |
public enum Funding { |
|
51 |
FP7, WT, FCT, H2020, NHMRC, ARC, SFI, MZOS, HRZZ, NWO, MESTD, FWF, SNSF, RCUK |
|
52 |
} |
|
53 |
|
|
54 | 46 |
@RequestMapping(value = "/openaire/export/**/project/dspace.do", method = RequestMethod.GET) |
55 | 47 |
void processDspace(final HttpServletRequest request, final ServletResponse response, |
56 | 48 |
@RequestParam(value = "startFrom", required = false) final String startFrom, |
... | ... | |
102 | 94 |
@RequestParam(value = "funding", required = true) final String funding, |
103 | 95 |
@RequestParam(value = "article293", required = false) final Boolean article293) throws Exception { |
104 | 96 |
|
105 |
final Funding requestedFunding = Funding.valueOf(funding.toUpperCase()); |
|
106 |
final String fundingPrefix = getFundingPrefix(requestedFunding, null); |
|
97 |
final String fundingPrefix = getFundingPrefix(funding, null); |
|
107 | 98 |
|
108 | 99 |
final String date = new SimpleDateFormat("yyyyMMdd").format(new Date()); |
109 | 100 |
final String filename = "projects_" + funding + "_" + date + ".tsv"; |
... | ... | |
184 | 175 |
protected String obtainQuery(final ProjectQueryParams params) throws IllegalArgumentException, IOException { |
185 | 176 |
String funding = params.getFundingProgramme(); |
186 | 177 |
String suffix = params.getFundingPath(); |
187 |
String fundingPrefix = getFundingPrefix(Funding.valueOf(funding.toUpperCase()), suffix); |
|
178 |
|
|
188 | 179 |
final StringTemplate st = new StringTemplate(IOUtils.toString(config.getProject().getProjectsFundingQueryTemplate().getInputStream(), UTF8)); |
189 |
st.setAttribute("fundingprefix", fundingPrefix);
|
|
180 |
st.setAttribute("fundingprefix", getFundingPrefix(funding, suffix));
|
|
190 | 181 |
String theQuery = setDateParameters(st.toString(), params); |
191 | 182 |
log.debug("Generated query: " + theQuery); |
192 | 183 |
return theQuery; |
193 | 184 |
} |
194 | 185 |
|
195 |
/** |
|
196 |
* @deprecated call the yet to come service that returns the funding prefix given a funding code |
|
197 |
*/ |
|
198 |
@Deprecated |
|
199 |
private String getFundingPrefix(final Funding funding, final String suffix) throws IllegalArgumentException { |
|
200 |
switch (funding) { |
|
201 |
case FCT: |
|
202 |
if (StringUtils.isBlank(suffix)) return "fct_________::FCT"; |
|
203 |
else return "fct_________::FCT::" + suffix; |
|
204 |
case WT: |
|
205 |
if (StringUtils.isBlank(suffix)) return "wt__________::WT"; |
|
206 |
else return "wt__________::WT::" + suffix; |
|
207 |
case FP7: |
|
208 |
if (StringUtils.isBlank(suffix)) return "ec__________::EC::FP7"; |
|
209 |
else return "ec__________::EC::FP7::" + suffix; |
|
210 |
case H2020: |
|
211 |
if (StringUtils.isBlank(suffix)) return "ec__________::EC::H2020"; |
|
212 |
else return "ec__________::EC::H2020" + suffix; |
|
213 |
case NHMRC: |
|
214 |
if (StringUtils.isBlank(suffix)) return "nhmrc_______::NHMRC"; |
|
215 |
else return "nhmrc_______::NHMRC::" + suffix; |
|
216 |
case ARC: |
|
217 |
if (StringUtils.isBlank(suffix)) return "arc_________::ARC"; |
|
218 |
else return "arc_________::ARC::" + suffix; |
|
219 |
case SFI: |
|
220 |
if (StringUtils.isBlank(suffix)) return "sfi_________::SFI"; |
|
221 |
else return "sfi_________::SFI::" + suffix; |
|
222 |
case MZOS: |
|
223 |
if (StringUtils.isBlank(suffix)) return "irb_hr______::MZOS"; |
|
224 |
else return "irb_hr______::MZOS::" + suffix; |
|
225 |
case HRZZ: |
|
226 |
if (StringUtils.isBlank(suffix)) return "irb_hr______::HRZZ"; |
|
227 |
else return "irb_hr______::HRZZ::" + suffix; |
|
228 |
case NWO: |
|
229 |
if (StringUtils.isBlank(suffix)) return "nwo_________::NWO"; |
|
230 |
else return "nwo_________::NWO::" + suffix; |
|
231 |
case MESTD: |
|
232 |
if (StringUtils.isBlank(suffix)) return "mestd_______::MESTD"; |
|
233 |
else return "mestd_______::MESTD::" + suffix; |
|
234 |
case FWF: |
|
235 |
if (StringUtils.isBlank(suffix)) return "fwf_________::FWF"; |
|
236 |
else return "fwf_________::FWF::" + suffix; |
|
237 |
case SNSF: |
|
238 |
if (StringUtils.isBlank(suffix)) return "snsf________::SNSF"; |
|
239 |
else return "snsf________::SNSF::" + suffix; |
|
240 |
case RCUK: |
|
241 |
if (StringUtils.isBlank(suffix)) return "rcuk________::RCUK"; |
|
242 |
else return "rcuk________::RCUK::" + suffix; |
|
243 |
default: |
|
244 |
throw new IllegalArgumentException("Invalid funding " + funding + " (valid are: " + Arrays.asList(Funding.values()) + ") "); |
|
186 |
private String getFundingPrefix(final String funding, final String suffix) { |
|
187 |
final Map<String, String> fundingIds = dao.readFundingpathIds(); |
|
188 |
if (!fundingIds.containsKey(funding.toUpperCase())) { |
|
189 |
throw new IllegalArgumentException("invalid funding " + funding); |
|
245 | 190 |
} |
191 |
String fundingPrefix = fundingIds.get(funding.toUpperCase()); |
|
192 |
return StringUtils.isBlank(suffix) ? fundingPrefix : fundingPrefix + "::" + suffix.toUpperCase(); |
|
246 | 193 |
} |
247 | 194 |
|
248 | 195 |
private String setDateParameters(final String query, final ProjectQueryParams params) { |
Also available in: Unified diff
removed deprecated stuff