Revision 47587
Added by Tsampikos Livisianos almost 7 years ago
modules/uoa-stats-service/trunk/deploy.info | ||
---|---|---|
1 |
{ |
|
2 |
"type_source": "SVN", |
|
3 |
"goal": "package -U -T 4C source:jar", |
|
4 |
"url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/uoa-stats-service/trunk", |
|
5 |
"deploy_repository": "dnet4-snapshots", |
|
6 |
"version": "4", |
|
7 |
"mail": "antleb@di.uoa.gr, kiatrop@di.uoa.gr, gkatsari@di.uoa.gr", |
|
8 |
"deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", |
|
9 |
"name": "uoa-stats-service" |
|
10 |
} |
modules/uoa-stats-service/trunk/src/main/java/eu/dnetlib/data/statsmanager/Query.java | ||
---|---|---|
1 |
package eu.dnetlib.data.statsmanager; |
|
2 |
|
|
3 |
import org.apache.log4j.Logger; |
|
4 |
|
|
5 |
import javax.xml.bind.annotation.XmlRootElement; |
|
6 |
|
|
7 |
|
|
8 |
@XmlRootElement(name = "Query") |
|
9 |
public class Query { |
|
10 |
private String id; |
|
11 |
private String name; |
|
12 |
private int dbResult; |
|
13 |
private int cqlResult; |
|
14 |
private statusFlag status; |
|
15 |
|
|
16 |
private Logger log = Logger.getLogger(this.getClass()); |
|
17 |
|
|
18 |
public Query() {}; |
|
19 |
|
|
20 |
public Query(String id, String key, Integer dbResult, int cqlResult, boolean qStatus) { |
|
21 |
this.id = id; |
|
22 |
this.name = key; |
|
23 |
this.dbResult = dbResult; |
|
24 |
this.cqlResult = cqlResult; |
|
25 |
if (qStatus) { |
|
26 |
this.status = statusFlag.success; |
|
27 |
|
|
28 |
} else { |
|
29 |
this.status = statusFlag.fail; |
|
30 |
} |
|
31 |
|
|
32 |
// log.debug("Created Query: " + this.toString()); |
|
33 |
} |
|
34 |
|
|
35 |
public enum statusFlag {success, fail} |
|
36 |
|
|
37 |
|
|
38 |
public String getName() { |
|
39 |
return name; |
|
40 |
} |
|
41 |
|
|
42 |
public void setName(String name) { |
|
43 |
this.name = name; |
|
44 |
} |
|
45 |
|
|
46 |
|
|
47 |
public int getDbResult() { |
|
48 |
return dbResult; |
|
49 |
} |
|
50 |
|
|
51 |
public void setDbResult(int dbResult) { |
|
52 |
this.dbResult = dbResult; |
|
53 |
} |
|
54 |
|
|
55 |
public int getCqlResult() { |
|
56 |
return cqlResult; |
|
57 |
} |
|
58 |
|
|
59 |
public void setCqlResult(int cqlResult) { |
|
60 |
this.cqlResult = cqlResult; |
|
61 |
} |
|
62 |
|
|
63 |
public String getId() { |
|
64 |
return id; |
|
65 |
} |
|
66 |
|
|
67 |
public void setId(String id) { |
|
68 |
this.id = id; |
|
69 |
} |
|
70 |
|
|
71 |
public statusFlag getStatus() { |
|
72 |
return status; |
|
73 |
} |
|
74 |
|
|
75 |
public void setStatus(statusFlag status) { |
|
76 |
this.status = status; |
|
77 |
} |
|
78 |
|
|
79 |
@Override |
|
80 |
public String toString() { |
|
81 |
return "Query{" + |
|
82 |
"id='" + id + '\'' + |
|
83 |
", name='" + name + '\'' + |
|
84 |
", dbResult=" + dbResult + |
|
85 |
", cqlResult=" + cqlResult + |
|
86 |
", status=" + status + |
|
87 |
'}'; |
|
88 |
} |
|
89 |
} |
modules/uoa-stats-service/trunk/src/main/java/eu/dnetlib/data/statsmanager/Validator.java | ||
---|---|---|
1 |
package eu.dnetlib.data.statsmanager; |
|
2 |
|
|
3 |
|
|
4 |
import eu.dnetlib.clients.data.search.ws.SearchWebService; |
|
5 |
import eu.dnetlib.common.rmi.UnimplementedException; |
|
6 |
import eu.dnetlib.domain.data.SearchResult; |
|
7 |
import org.apache.cxf.jaxws.JaxWsProxyFactoryBean; |
|
8 |
import org.apache.log4j.Logger; |
|
9 |
import org.springframework.core.io.Resource; |
|
10 |
|
|
11 |
import javax.sql.DataSource; |
|
12 |
import java.sql.*; |
|
13 |
import java.util.HashMap; |
|
14 |
import java.util.Map; |
|
15 |
import java.util.Properties; |
|
16 |
|
|
17 |
|
|
18 |
public class Validator { |
|
19 |
|
|
20 |
private ValidationReport validationReport; |
|
21 |
private Resource validationQueriesFile; |
|
22 |
private DataSource dataSource; |
|
23 |
private String historySchema; |
|
24 |
private String shadowSearchURL; |
|
25 |
private static SearchWebService searchWebService = null; |
|
26 |
private Logger log = Logger.getLogger(this.getClass()); |
|
27 |
|
|
28 |
public Validator() { |
|
29 |
} |
|
30 |
|
|
31 |
private void initSearch() { |
|
32 |
if (searchWebService == null) { |
|
33 |
JaxWsProxyFactoryBean factory = new JaxWsProxyFactoryBean(); |
|
34 |
factory.setServiceClass(SearchWebService.class); |
|
35 |
factory.setAddress(shadowSearchURL); |
|
36 |
searchWebService = (SearchWebService) factory.create(); |
|
37 |
} |
|
38 |
} |
|
39 |
|
|
40 |
public ValidationReport validateDatabase() throws Exception { |
|
41 |
ValidationReport report = new ValidationReport(); |
|
42 |
Properties p = new Properties(); |
|
43 |
|
|
44 |
this.validationReport = report; |
|
45 |
|
|
46 |
try { //TODO initiate search |
|
47 |
initSearch(); |
|
48 |
log.debug("Search Client initiated at " + this.shadowSearchURL); |
|
49 |
} catch (Exception e) { |
|
50 |
log.error(e.toString()); |
|
51 |
throw new Exception(e); |
|
52 |
|
|
53 |
} |
|
54 |
|
|
55 |
try { |
|
56 |
|
|
57 |
p.loadFromXML(validationQueriesFile.getInputStream()); |
|
58 |
|
|
59 |
} catch (Exception e) { |
|
60 |
log.error("Error while loading queries file :" + e.toString()); |
|
61 |
throw new Exception(e); |
|
62 |
|
|
63 |
} |
|
64 |
|
|
65 |
|
|
66 |
int queryCount = Integer.parseInt(p.getProperty("queries.count")); |
|
67 |
log.info("Query Count: " + queryCount + "\n"); |
|
68 |
|
|
69 |
for (int i = 1; i <= queryCount; i++) { |
|
70 |
String schema = p.getProperty("queries.schema"); |
|
71 |
|
|
72 |
String queryType = p.getProperty("queries." + i + ".type"); |
|
73 |
String sql = p.getProperty("queries." + i + ".sql").replaceAll("schema", schema); |
|
74 |
String cql = p.getProperty("queries." + i + ".cql.query"); |
|
75 |
String name = p.getProperty("queries." + i + ".name"); |
|
76 |
String id = p.getProperty("queries." + i + ".id"); |
|
77 |
|
|
78 |
log.debug("Query: id " + id + " " + name + "\n" + " SQL " + sql + "\n" + "CQL" + cql + "\n"); |
|
79 |
if (queryType.equals("search")) { |
|
80 |
|
|
81 |
|
|
82 |
int sqlResult = executeSQLCountQuery(sql); |
|
83 |
int cqlResult = executeCQLCountQuery(cql); |
|
84 |
|
|
85 |
boolean status = validate(sqlResult, cqlResult); |
|
86 |
saveResult(id, name, sql, String.valueOf(sqlResult), validationReport.getValidationDate().toString()); |
|
87 |
validationReport.addQuery(id, name, sqlResult, cqlResult, status); |
|
88 |
|
|
89 |
|
|
90 |
} else if (queryType.equals("browse")) { |
|
91 |
|
|
92 |
// Map<String, Integer> sqlResults = executeSQLBrowseQuery(sql); |
|
93 |
// Map<String, Integer> cqlResults = executeCQLBrowseQuery(cql, p.getProperty("queries." + i + ".cql.groupBy")); |
|
94 |
|
|
95 |
// TODO compare and update report |
|
96 |
} |
|
97 |
} |
|
98 |
|
|
99 |
log.info("Generated report : " + report.toString()); |
|
100 |
return report; |
|
101 |
} |
|
102 |
|
|
103 |
//@Transactional |
|
104 |
public void saveResult(String id, String desc, String qString, String result, String date) throws Exception { |
|
105 |
Connection con = dataSource.getConnection(); |
|
106 |
|
|
107 |
log.info("Writing report to DB " + con.getMetaData().getURL() + " and historySchema " + this.historySchema + "..."); |
|
108 |
|
|
109 |
String q = "UPDATE " + this.historySchema + ".query SET query_id=?,query=?, description=? WHERE query_id=? ;" + |
|
110 |
"INSERT INTO " + this.historySchema + ".query (query_id, query, description) SELECT ? ,?,? WHERE NOT EXISTS" + |
|
111 |
"(SELECT 1 FROM " + this.historySchema + ".query WHERE query_id=?)"; |
|
112 |
|
|
113 |
// log.info("Executing query update " + q); |
|
114 |
|
|
115 |
PreparedStatement stm = con.prepareStatement(q); |
|
116 |
|
|
117 |
|
|
118 |
stm.setString(1, id); |
|
119 |
stm.setString(2, qString); |
|
120 |
stm.setString(3, desc); |
|
121 |
stm.setString(4, id); |
|
122 |
stm.setString(5, id); |
|
123 |
stm.setString(6, qString); |
|
124 |
stm.setString(7, desc); |
|
125 |
stm.setString(8, id); |
|
126 |
|
|
127 |
stm.executeUpdate(); |
|
128 |
|
|
129 |
// q = "INSERT INTO value(query_id, num, date) values ('" + id + "','" + result + "','" + date + "');"; |
|
130 |
|
|
131 |
q = "INSERT INTO " + this.historySchema + ".value(query_id, num, date) values (?,?,?);"; |
|
132 |
stm.clearParameters(); |
|
133 |
|
|
134 |
|
|
135 |
stm = con.prepareStatement(q); |
|
136 |
stm.setString(1, id); |
|
137 |
stm.setString(2, result); |
|
138 |
stm.setString(3, date); |
|
139 |
|
|
140 |
//log.info("Executing query update " + q); |
|
141 |
stm.executeUpdate(); |
|
142 |
|
|
143 |
stm.close(); |
|
144 |
con.close(); |
|
145 |
} |
|
146 |
|
|
147 |
|
|
148 |
private boolean validate(int sqlResult, int cqlResult) { |
|
149 |
|
|
150 |
log.debug(" Validation input -> sql : " + sqlResult + " cql : " + cqlResult); |
|
151 |
if (sqlResult == cqlResult) { |
|
152 |
|
|
153 |
return true; |
|
154 |
} |
|
155 |
|
|
156 |
return false; |
|
157 |
} |
|
158 |
|
|
159 |
private Map<String, Integer> executeCQLBrowseQuery(String cql, String groupBy) { |
|
160 |
// TODO implement me please |
|
161 |
|
|
162 |
//oaftype=result and deletedbyinferece=false and type=publication , groupby = access_mode |
|
163 |
// searchServiceServiceLocator.getService().refine(); |
|
164 |
|
|
165 |
//public SearchResult refine(String queryText, String transformer, |
|
166 |
//String locale, Collection<String> fields) throws SearchServiceException; |
|
167 |
|
|
168 |
// public SearchResult refine(String queryText, String transformer, |
|
169 |
// String locale, Collection<String> fields) throws SearchServiceException; |
|
170 |
//[2:52:39 PM] Antonis Lempesis: |
|
171 |
|
|
172 |
|
|
173 |
//query= oaftype=result and deletedbyinferece=false and type=publication |
|
174 |
|
|
175 |
//transformer = results_openaire |
|
176 |
|
|
177 |
// locale=UTF-8 |
|
178 |
|
|
179 |
// fields = access_mode |
|
180 |
|
|
181 |
|
|
182 |
// set to 1 page and results |
|
183 |
// public SearchResult search(String queryText, String transformer, |
|
184 |
// String locale, int page, int size) throws SearchServiceException; |
|
185 |
|
|
186 |
throw new UnimplementedException(); |
|
187 |
} |
|
188 |
|
|
189 |
private Map<String, Integer> executeSQLBrowseQuery(String sql) { |
|
190 |
// TODO implement me please |
|
191 |
throw new UnimplementedException(); |
|
192 |
} |
|
193 |
|
|
194 |
private int executeCQLCountQuery(String cql) throws Exception { |
|
195 |
try { |
|
196 |
if (cql == null || cql.isEmpty() || cql.equalsIgnoreCase("not available")) { |
|
197 |
return 0; |
|
198 |
} |
|
199 |
SearchResult result = searchWebService.search(cql, "results_openaire", "en_GB", 1, 1); |
|
200 |
|
|
201 |
return result.getTotal(); |
|
202 |
|
|
203 |
|
|
204 |
} catch (Exception e) { |
|
205 |
log.error("Could not execute CQL query. Reason: " + e); |
|
206 |
throw new Exception("Could not execute CQL query. Reason: ", e); |
|
207 |
} |
|
208 |
} |
|
209 |
|
|
210 |
|
|
211 |
private int executeSQLCountQuery(String sql) throws Exception { |
|
212 |
|
|
213 |
Connection con = null; |
|
214 |
try { |
|
215 |
|
|
216 |
con = dataSource.getConnection(); |
|
217 |
|
|
218 |
|
|
219 |
Statement st = con.createStatement(); |
|
220 |
if (st.execute(sql)) { |
|
221 |
|
|
222 |
ResultSet rs = st.getResultSet(); |
|
223 |
int res = getResult(rs); |
|
224 |
st.close(); |
|
225 |
return res; |
|
226 |
} else { |
|
227 |
log.error("Fail to execute command " + sql + " " + st.getWarnings()); |
|
228 |
throw new Exception("Fail to execute command " + sql + " " + st.getWarnings()); |
|
229 |
} |
|
230 |
|
|
231 |
|
|
232 |
} catch (Exception e) { |
|
233 |
log.error("Could not execute sql query " + sql + " : " + e); |
|
234 |
throw new Exception("Could not execute sql query " + sql + " : ", e); |
|
235 |
} finally { |
|
236 |
if (con != null) { |
|
237 |
con.close(); |
|
238 |
|
|
239 |
|
|
240 |
} |
|
241 |
} |
|
242 |
|
|
243 |
} |
|
244 |
|
|
245 |
private int getResult(ResultSet rs) throws Exception { |
|
246 |
HashMap<String, Integer> data = new HashMap<String, Integer>(); |
|
247 |
int res = -1; |
|
248 |
try { |
|
249 |
|
|
250 |
ResultSetMetaData rsmd = rs.getMetaData(); |
|
251 |
|
|
252 |
if (rs.next()) { |
|
253 |
|
|
254 |
res = (Integer.valueOf(rs.getString(1))); |
|
255 |
rs.close(); |
|
256 |
} |
|
257 |
|
|
258 |
return res; |
|
259 |
} catch (Exception e) { |
|
260 |
log.error("Could not process results :" + e); |
|
261 |
throw new Exception("Could not process results :", e); |
|
262 |
} |
|
263 |
|
|
264 |
} |
|
265 |
|
|
266 |
private HashMap<String, Integer> getResults(ResultSet rs) throws Exception { |
|
267 |
HashMap<String, Integer> data = new HashMap<String, Integer>(); |
|
268 |
try { |
|
269 |
|
|
270 |
ResultSetMetaData rsmd = rs.getMetaData(); |
|
271 |
while (rs.next()) { |
|
272 |
for (int i = 1; i < rsmd.getColumnCount() - 1; i++) { |
|
273 |
data.put(rsmd.getColumnName(i), rs.getInt(i)); |
|
274 |
} |
|
275 |
|
|
276 |
} |
|
277 |
|
|
278 |
return data; |
|
279 |
|
|
280 |
} catch (Exception e) { |
|
281 |
log.error("Could not process results :" + e); |
|
282 |
throw new Exception("Could not process results :", e); |
|
283 |
} |
|
284 |
|
|
285 |
} |
|
286 |
|
|
287 |
|
|
288 |
public ValidationReport getValidationReport() { |
|
289 |
return validationReport; |
|
290 |
} |
|
291 |
|
|
292 |
public void setValidationReport(ValidationReport validationReport) { |
|
293 |
this.validationReport = validationReport; |
|
294 |
} |
|
295 |
|
|
296 |
public Resource getValidationQueriesFile() { |
|
297 |
return validationQueriesFile; |
|
298 |
} |
|
299 |
|
|
300 |
public void setValidationQueriesFile(Resource validationQueriesFile) { |
|
301 |
this.validationQueriesFile = validationQueriesFile; |
|
302 |
} |
|
303 |
|
|
304 |
public DataSource getDataSource() { |
|
305 |
return dataSource; |
|
306 |
} |
|
307 |
|
|
308 |
public void setDataSource(DataSource dataSource) { |
|
309 |
this.dataSource = dataSource; |
|
310 |
} |
|
311 |
|
|
312 |
public String getShadowSearchURL() { |
|
313 |
return shadowSearchURL; |
|
314 |
} |
|
315 |
|
|
316 |
public void setShadowSearchURL(String shadowSearchURL) { |
|
317 |
this.shadowSearchURL = shadowSearchURL; |
|
318 |
} |
|
319 |
|
|
320 |
public static SearchWebService getSearchWebService() { |
|
321 |
return searchWebService; |
|
322 |
} |
|
323 |
|
|
324 |
public static void setSearchWebService(SearchWebService searchWebService) { |
|
325 |
Validator.searchWebService = searchWebService; |
|
326 |
} |
|
327 |
|
|
328 |
public String getHistorySchema() { |
|
329 |
return historySchema; |
|
330 |
} |
|
331 |
|
|
332 |
public void setHistorySchema(String historySchema) { |
|
333 |
this.historySchema = historySchema; |
|
334 |
} |
|
335 |
|
|
336 |
|
|
337 |
} |
modules/uoa-stats-service/trunk/src/main/java/eu/dnetlib/data/statsmanager/ValidationReport.java | ||
---|---|---|
1 |
package eu.dnetlib.data.statsmanager; |
|
2 |
|
|
3 |
import org.apache.log4j.Logger; |
|
4 |
|
|
5 |
import javax.xml.bind.annotation.XmlRootElement; |
|
6 |
import java.text.DateFormat; |
|
7 |
import java.text.SimpleDateFormat; |
|
8 |
import java.util.ArrayList; |
|
9 |
import java.util.Date; |
|
10 |
import java.util.HashMap; |
|
11 |
import java.util.Map; |
|
12 |
|
|
13 |
/** |
|
14 |
* Created by antleb on 10/25/14. |
|
15 |
*/ |
|
16 |
|
|
17 |
|
|
18 |
@XmlRootElement(name = "ValidationReport") |
|
19 |
public class ValidationReport { |
|
20 |
|
|
21 |
private ArrayList<Query> Queries; |
|
22 |
|
|
23 |
private Date validationDate; |
|
24 |
private Logger log = Logger.getLogger(this.getClass()); |
|
25 |
|
|
26 |
public ValidationReport() { |
|
27 |
DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); |
|
28 |
this.validationDate = new Date(); |
|
29 |
Queries = new ArrayList<Query>(); |
|
30 |
} |
|
31 |
|
|
32 |
|
|
33 |
public void addQuery(String id,String name, int sqlResult, int cqlResult, boolean status) |
|
34 |
|
|
35 |
{ |
|
36 |
Query q = new Query(id,name, sqlResult, cqlResult, status); |
|
37 |
this.getQueries().add(q); |
|
38 |
|
|
39 |
|
|
40 |
} |
|
41 |
/* |
|
42 |
DB validation report. Contains: |
|
43 |
- queries: |
|
44 |
- query name |
|
45 |
- db result |
|
46 |
- cql result |
|
47 |
- status (success/fail) |
|
48 |
- metadata: |
|
49 |
- validation date |
|
50 |
- ??? |
|
51 |
*/ |
|
52 |
|
|
53 |
public ArrayList<Query> getQueries() { |
|
54 |
return Queries; |
|
55 |
} |
|
56 |
|
|
57 |
public void setQueries(ArrayList<Query> queries) { |
|
58 |
Queries = queries; |
|
59 |
} |
|
60 |
|
|
61 |
@Override |
|
62 |
public String toString() { |
|
63 |
return "ValidationReport{" + |
|
64 |
"Queries=" + Queries + |
|
65 |
", validationDate=" + validationDate + |
|
66 |
'}'; |
|
67 |
} |
|
68 |
|
|
69 |
public Date getValidationDate() { |
|
70 |
return validationDate; |
|
71 |
} |
|
72 |
|
|
73 |
public void setValidationDate(Date validationDate) { |
|
74 |
this.validationDate = validationDate; |
|
75 |
} |
|
76 |
} |
modules/uoa-stats-service/trunk/src/main/java/eu/dnetlib/data/statsmanager/CacheController.java | ||
---|---|---|
1 |
package eu.dnetlib.data.statsmanager; |
|
2 |
|
|
3 |
|
|
4 |
import org.apache.log4j.Logger; |
|
5 |
import org.apache.tools.ant.util.Base64Converter; |
|
6 |
|
|
7 |
import javax.net.ssl.HttpsURLConnection; |
|
8 |
import java.io.BufferedReader; |
|
9 |
import java.io.InputStreamReader; |
|
10 |
import java.net.URL; |
|
11 |
import java.net.URLConnection; |
|
12 |
import java.util.HashMap; |
|
13 |
import java.util.Map; |
|
14 |
|
|
15 |
|
|
16 |
public class CacheController { |
|
17 |
|
|
18 |
private HashMap<String, URL> actions; |
|
19 |
|
|
20 |
private Logger log = Logger.getLogger(this.getClass()); |
|
21 |
|
|
22 |
private String httpsCredentials; |
|
23 |
|
|
24 |
public CacheController() { |
|
25 |
} |
|
26 |
|
|
27 |
public void executeCommand(String action, Map<String, String> parameters) throws Exception { |
|
28 |
|
|
29 |
|
|
30 |
/*if (cacheURL != null && !cacheURL.isEmpty()) { |
|
31 |
log.debug("existing cache url " + this.getActions().get(action)); |
|
32 |
String actionUrl = this.getActions().get(action).toString(); |
|
33 |
actionUrl = actionUrl.substring(actionUrl.lastIndexOf("/"), actionUrl.length()); |
|
34 |
url = new URL(cacheURL + actionUrl); |
|
35 |
log.info("creds for url " + url + " " + credentials); |
|
36 |
executeRemoteScript(url, credentials); |
|
37 |
|
|
38 |
} else { |
|
39 |
*/ |
|
40 |
|
|
41 |
String urlString = this.actions.get(action).toString(); |
|
42 |
for (Map.Entry<String, String> e : parameters.entrySet()) { |
|
43 |
if (!e.getKey().equals("cache")&&!e.getKey().equals("error")) { |
|
44 |
urlString += "&" + e.getKey() + "=" + e.getValue(); |
|
45 |
|
|
46 |
|
|
47 |
} |
|
48 |
} |
|
49 |
// credentials = this.cacheCredInfo.get(url.toString().substring(0, url.toString().lastIndexOf("/"))); |
|
50 |
executeRemoteScript(new URL(urlString)); |
|
51 |
} |
|
52 |
|
|
53 |
private void executeRemoteScript(URL url) throws Exception { |
|
54 |
try { |
|
55 |
|
|
56 |
|
|
57 |
Base64Converter converter = new Base64Converter(); |
|
58 |
BufferedReader in = null; |
|
59 |
|
|
60 |
if (url.toString().startsWith("https://")) { |
|
61 |
|
|
62 |
String encoding = converter.encode(httpsCredentials.getBytes("UTF-8")); |
|
63 |
|
|
64 |
log.debug("Using https : " + url.toString()); |
|
65 |
HttpsURLConnection yc = (HttpsURLConnection) url.openConnection(); |
|
66 |
yc.setRequestProperty("Authorization", String.format("Basic %s", encoding)); |
|
67 |
in = new BufferedReader( |
|
68 |
new InputStreamReader( |
|
69 |
yc.getInputStream())); |
|
70 |
} else { |
|
71 |
log.debug("Using normal url : " + url.toString()); |
|
72 |
URLConnection yc = (URLConnection) url.openConnection(); |
|
73 |
|
|
74 |
in |
|
75 |
= new BufferedReader( |
|
76 |
new InputStreamReader( |
|
77 |
yc.getInputStream())); |
|
78 |
|
|
79 |
} |
|
80 |
|
|
81 |
String inputLine; |
|
82 |
|
|
83 |
while ((inputLine = in.readLine()) != null) |
|
84 |
log.debug(inputLine); |
|
85 |
|
|
86 |
|
|
87 |
in.close(); |
|
88 |
|
|
89 |
|
|
90 |
} catch (Exception e) { |
|
91 |
log.error("Error while calling php script over http. Reason: " + e); |
|
92 |
throw new Exception(e); |
|
93 |
} |
|
94 |
} |
|
95 |
|
|
96 |
public HashMap<String, URL> getActions() { |
|
97 |
return actions; |
|
98 |
} |
|
99 |
|
|
100 |
public void setActions(HashMap<String, URL> actions) { |
|
101 |
this.actions = actions; |
|
102 |
} |
|
103 |
|
|
104 |
public CacheController(HashMap<String, URL> actions) { |
|
105 |
this.actions = actions; |
|
106 |
} |
|
107 |
|
|
108 |
|
|
109 |
public String getHttpsCredentials() { |
|
110 |
return httpsCredentials; |
|
111 |
} |
|
112 |
|
|
113 |
public void setHttpsCredentials(String httpsCredentials) { |
|
114 |
this.httpsCredentials = httpsCredentials; |
|
115 |
} |
|
116 |
} |
modules/uoa-stats-service/trunk/src/main/java/eu/dnetlib/data/statsmanager/StatsManager.java | ||
---|---|---|
1 |
package eu.dnetlib.data.statsmanager; |
|
2 |
|
|
3 |
|
|
4 |
import org.apache.log4j.Logger; |
|
5 |
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob; |
|
6 |
import eu.dnetlib.enabling.tools.blackboard.BlackboardNotificationHandler; |
|
7 |
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler; |
|
8 |
|
|
9 |
import org.springframework.transaction.annotation.Transactional; |
|
10 |
|
|
11 |
import javax.sql.DataSource; |
|
12 |
import java.sql.Connection; |
|
13 |
import java.util.HashMap; |
|
14 |
import java.util.Map; |
|
15 |
|
|
16 |
public class StatsManager { |
|
17 |
|
|
18 |
private Validator validator; |
|
19 |
|
|
20 |
private ValidationReport validationReport; |
|
21 |
|
|
22 |
private DataSource dataSource; |
|
23 |
private CacheController prodCacheController; |
|
24 |
private CacheController testCacheController; |
|
25 |
|
|
26 |
private Logger log = Logger.getLogger(this.getClass()); |
|
27 |
|
|
28 |
public StatsManager() { |
|
29 |
|
|
30 |
} |
|
31 |
|
|
32 |
|
|
33 |
public ValidationReport validateDatabase() throws Exception { |
|
34 |
|
|
35 |
this.validationReport = validator.validateDatabase(); |
|
36 |
|
|
37 |
|
|
38 |
return validationReport; |
|
39 |
} |
|
40 |
|
|
41 |
|
|
42 |
@Transactional |
|
43 |
public void promoteShadowSchema() throws Exception { |
|
44 |
Connection con = dataSource.getConnection(); |
|
45 |
|
|
46 |
log.info("Backing up and Replacing public schema with shadow in " + dataSource.getConnection().getMetaData().getURL() + " ..."); |
|
47 |
con.createStatement().execute("drop schema if exists backup CASCADE ;"); |
|
48 |
//con.createStatement().execute("alter schema public rename to backup ;"); |
|
49 |
con.createStatement().execute("drop schema if exists public CASCADE ;"); |
|
50 |
con.createStatement().execute("alter schema shadow rename TO public ;"); |
|
51 |
log.info("All ops done!"); |
|
52 |
con.close(); |
|
53 |
} |
|
54 |
|
|
55 |
public void executeCacheAction(BlackboardJob job) throws Exception { |
|
56 |
|
|
57 |
if (job.getAction().equals("refreshCache")) { |
|
58 |
|
|
59 |
executeCommand("refreshAll", job.getParameters()); |
|
60 |
} else if (job.getAction().equals("refreshCharts")) { |
|
61 |
|
|
62 |
executeCommand(job.getAction(), job.getParameters()); |
|
63 |
} else if (job.getAction().equals("refreshNums")) { |
|
64 |
|
|
65 |
executeCommand(job.getAction(), job.getParameters()); |
|
66 |
} else if (job.getAction().equals("promoteShadow")) { |
|
67 |
|
|
68 |
executeCommand("promoteAll", job.getParameters()); |
|
69 |
this.promoteShadowSchema(); |
|
70 |
|
|
71 |
} else if (job.getAction().equals("promoteNums")) { |
|
72 |
|
|
73 |
executeCommand(job.getAction(), job.getParameters()); |
|
74 |
} else if (job.getAction().equals("promoteCharts")) { |
|
75 |
|
|
76 |
executeCommand(job.getAction(), job.getParameters()); |
|
77 |
} else if (job.getAction().equals("promoteCache")) { |
|
78 |
|
|
79 |
executeCommand("promoteAll", job.getParameters()); |
|
80 |
} else if (job.getAction().equals("restore")) { |
|
81 |
|
|
82 |
executeCommand(job.getAction(), job.getParameters()); |
|
83 |
} else if (job.getAction().equals("migrate")) { |
|
84 |
|
|
85 |
executeCommand(job.getAction(), job.getParameters()); |
|
86 |
} else if (job.getAction().equals("backup")) { |
|
87 |
|
|
88 |
executeCommand(job.getAction(), job.getParameters()); |
|
89 |
|
|
90 |
} else { |
|
91 |
log.error("Wrong action given "); |
|
92 |
throw new Exception("Wrong action given "); |
|
93 |
} |
|
94 |
log.info("Done ! "); |
|
95 |
} |
|
96 |
|
|
97 |
private void executeCommand(String action, Map<String, String> parameters) throws Exception { |
|
98 |
|
|
99 |
String cache = parameters.get("cache"); |
|
100 |
log.info("Executing action " + action + " in " + cache + "..."); |
|
101 |
if (cache == null || cache.contains("test")) { |
|
102 |
testCacheController.executeCommand(action, parameters); |
|
103 |
} else if (cache.contains("beta") || (cache.contains("production"))) { |
|
104 |
prodCacheController.executeCommand(action, parameters); |
|
105 |
} else { |
|
106 |
throw new Exception("Wrong cache id"); |
|
107 |
} |
|
108 |
} |
|
109 |
|
|
110 |
|
|
111 |
public DataSource getDataSource() { |
|
112 |
return dataSource; |
|
113 |
} |
|
114 |
|
|
115 |
public void setDataSource(DataSource dataSource) { |
|
116 |
this.dataSource = dataSource; |
|
117 |
} |
|
118 |
|
|
119 |
|
|
120 |
public Validator getValidator() { |
|
121 |
return validator; |
|
122 |
} |
|
123 |
|
|
124 |
public void setValidator(Validator validator) { |
|
125 |
this.validator = validator; |
|
126 |
} |
|
127 |
|
|
128 |
public ValidationReport getValidationReport() { |
|
129 |
return validationReport; |
|
130 |
} |
|
131 |
|
|
132 |
public void setValidationReport(ValidationReport validationReport) { |
|
133 |
this.validationReport = validationReport; |
|
134 |
} |
|
135 |
|
|
136 |
public CacheController getProdCacheController() { |
|
137 |
return prodCacheController; |
|
138 |
} |
|
139 |
|
|
140 |
public void setProdCacheController(CacheController prodCacheController) { |
|
141 |
this.prodCacheController = prodCacheController; |
|
142 |
} |
|
143 |
|
|
144 |
public CacheController getTestCacheController() { |
|
145 |
return testCacheController; |
|
146 |
} |
|
147 |
|
|
148 |
public void setTestCacheController(CacheController testCacheController) { |
|
149 |
this.testCacheController = testCacheController; |
|
150 |
} |
|
151 |
} |
modules/uoa-stats-service/trunk/src/main/java/eu/dnetlib/data/statsmanager/StatsManagerServiceImpl.java | ||
---|---|---|
1 |
package eu.dnetlib.data.statsmanager; |
|
2 |
|
|
3 |
import eu.dnetlib.api.DriverService; |
|
4 |
import eu.dnetlib.api.data.StatsManagerService; |
|
5 |
import eu.dnetlib.domain.ActionType; |
|
6 |
import eu.dnetlib.domain.ResourceType; |
|
7 |
import eu.dnetlib.domain.enabling.Notification; |
|
8 |
import gr.uoa.di.driver.app.DriverServiceImpl; |
|
9 |
import gr.uoa.di.driver.enabling.issn.NotificationListener; |
|
10 |
|
|
11 |
/** |
|
12 |
* Created by antleb on 10/25/14. |
|
13 |
*/ |
|
14 |
public class StatsManagerServiceImpl extends DriverServiceImpl implements StatsManagerService { |
|
15 |
|
|
16 |
private StatsManagerServiceBlackboardHandler statsManagerServiceBlackboardHandler; |
|
17 |
|
|
18 |
public void init() { |
|
19 |
super.init(); |
|
20 |
|
|
21 |
this.subscribe( |
|
22 |
ActionType.UPDATE, |
|
23 |
ResourceType.STATSMANAGERSERVICERESOURCETYPE, |
|
24 |
this.getServiceEPR().getParameter("serviceId"), |
|
25 |
"RESOURCE_PROFILE/BODY/BLACKBOARD/LAST_REQUEST", |
|
26 |
new NotificationListener() { |
|
27 |
|
|
28 |
@Override |
|
29 |
public void processNotification(Notification notification) { |
|
30 |
statsManagerServiceBlackboardHandler.notified( |
|
31 |
notification.getSubscriptionId(), |
|
32 |
notification.getTopic(), |
|
33 |
notification.getIsId(), |
|
34 |
notification.getMessage()); |
|
35 |
} |
|
36 |
}); |
|
37 |
} |
|
38 |
|
|
39 |
public StatsManagerServiceBlackboardHandler getStatsManagerServiceBlackboardHandler() { |
|
40 |
return statsManagerServiceBlackboardHandler; |
|
41 |
} |
|
42 |
|
|
43 |
public void setStatsManagerServiceBlackboardHandler(StatsManagerServiceBlackboardHandler statsManagerServiceBlackboardHandler) { |
|
44 |
this.statsManagerServiceBlackboardHandler = statsManagerServiceBlackboardHandler; |
|
45 |
} |
|
46 |
} |
modules/uoa-stats-service/trunk/src/main/java/eu/dnetlib/data/statsmanager/StatsManagerController.java | ||
---|---|---|
1 |
package eu.dnetlib.data.statsmanager; |
|
2 |
|
|
3 |
|
|
4 |
import org.springframework.stereotype.Controller; |
|
5 |
import org.springframework.web.bind.annotation.RequestMapping; |
|
6 |
import org.springframework.web.bind.annotation.RequestMethod; |
|
7 |
|
|
8 |
import javax.servlet.http.HttpServletRequest; |
|
9 |
import javax.servlet.http.HttpServletResponse; |
|
10 |
import javax.xml.bind.JAXBContext; |
|
11 |
import javax.xml.bind.JAXBException; |
|
12 |
import java.io.IOException; |
|
13 |
|
|
14 |
/** |
|
15 |
* Created by antleb on 10/25/14. |
|
16 |
*/ |
|
17 |
@Controller |
|
18 |
public class StatsManagerController { |
|
19 |
|
|
20 |
private StatsManager statsManager; |
|
21 |
|
|
22 |
@RequestMapping(value = "/stats", method = RequestMethod.GET) |
|
23 |
public void getReport(HttpServletRequest request, HttpServletResponse response) throws IOException, JAXBException { |
|
24 |
ValidationReport report = statsManager.getValidationReport(); |
|
25 |
|
|
26 |
JAXBContext.newInstance().createMarshaller().marshal(report, response.getWriter()); |
|
27 |
} |
|
28 |
|
|
29 |
public StatsManager getStatsManager() { |
|
30 |
return statsManager; |
|
31 |
} |
|
32 |
|
|
33 |
public void setStatsManager(StatsManager statsManager) { |
|
34 |
this.statsManager = statsManager; |
|
35 |
} |
|
36 |
} |
modules/uoa-stats-service/trunk/src/main/java/eu/dnetlib/data/statsmanager/StatsManagerServiceBlackboardHandler.java | ||
---|---|---|
1 |
package eu.dnetlib.data.statsmanager; |
|
2 |
|
|
3 |
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob; |
|
4 |
import eu.dnetlib.enabling.tools.blackboard.BlackboardNotificationHandler; |
|
5 |
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler; |
|
6 |
import org.apache.log4j.Logger; |
|
7 |
|
|
8 |
import javax.xml.bind.JAXBContext; |
|
9 |
import java.io.PrintWriter; |
|
10 |
import java.io.StringWriter; |
|
11 |
|
|
12 |
/** |
|
13 |
* Created by antleb on 10/25/14. |
|
14 |
*/ |
|
15 |
public class StatsManagerServiceBlackboardHandler extends BlackboardNotificationHandler<BlackboardServerHandler> { |
|
16 |
|
|
17 |
private static Logger logger = Logger.getLogger(StatsManagerServiceBlackboardHandler.class); |
|
18 |
private StatsManager statsManager; |
|
19 |
|
|
20 |
protected void processJob(BlackboardJob job) { |
|
21 |
//TODO needs this? |
|
22 |
super.processJob(job); |
|
23 |
|
|
24 |
// logger.info("Beginning new job..."); |
|
25 |
// logger.info("Id: " + job.getId() + " Date : " + job.getDate()); |
|
26 |
String action = job.getAction(); |
|
27 |
|
|
28 |
logger.info("Stats Manager :Got BB message: " + action); |
|
29 |
|
|
30 |
try { |
|
31 |
getBlackboardHandler().ongoing(job); |
|
32 |
|
|
33 |
if (action.equals("validate")) { |
|
34 |
ValidationReport report = statsManager.validateDatabase(); |
|
35 |
StringWriter sw = new StringWriter(); |
|
36 |
JAXBContext.newInstance(ValidationReport.class).createMarshaller().marshal(report, sw); |
|
37 |
job.getParameters().put("report", sw.toString()); |
|
38 |
|
|
39 |
} |
|
40 |
else { |
|
41 |
statsManager.executeCacheAction(job); |
|
42 |
} |
|
43 |
|
|
44 |
|
|
45 |
getBlackboardHandler().done(job); |
|
46 |
} catch ( |
|
47 |
Exception e |
|
48 |
) |
|
49 |
|
|
50 |
{ |
|
51 |
getBlackboardHandler().failed(job, e); |
|
52 |
|
|
53 |
logger.error("Error processing job", e); |
|
54 |
} |
|
55 |
|
|
56 |
} |
|
57 |
|
|
58 |
|
|
59 |
public StatsManager getStatsManager() { |
|
60 |
return statsManager; |
|
61 |
} |
|
62 |
|
|
63 |
public void setStatsManager(StatsManager statsManager) { |
|
64 |
this.statsManager = statsManager; |
|
65 |
} |
|
66 |
} |
modules/uoa-stats-service/trunk/src/main/resources/eu/dnetlib/data/statsmanager/validationQueriesBeta.xml | ||
---|---|---|
1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
2 |
<!DOCTYPE properties |
|
3 |
SYSTEM "http://java.sun.com/dtd/properties.dtd"> |
|
4 |
<properties> |
|
5 |
|
|
6 |
<entry key="queries.count">38</entry> |
|
7 |
<entry key="queries.schema">shadow</entry> |
|
8 |
<!--RESULTS--> |
|
9 |
<entry key="queries.1.type">search</entry> |
|
10 |
<entry key="queries.1.sql">select count(*) as results from schema.result</entry> |
|
11 |
<entry key="queries.1.cql.query">oaftype=result and deletedbyinference=false</entry> |
|
12 |
<entry key="queries.1.name">Results </entry> |
|
13 |
<entry key="queries.1.id">res</entry> |
|
14 |
|
|
15 |
<!--PROJECTS--> |
|
16 |
|
|
17 |
<entry key="queries.2.type">search</entry> |
|
18 |
<entry key="queries.2.sql">SELECT count(*) FROM schema.project</entry> |
|
19 |
<entry key="queries.2.cql.query">oaftype=project</entry> |
|
20 |
<entry key="queries.2.name">Projects </entry> |
|
21 |
<entry key="queries.2.id">proj</entry> |
|
22 |
|
|
23 |
<!--PUBS--> |
|
24 |
<entry key="queries.3.type">search</entry> |
|
25 |
<entry key="queries.3.sql">SELECT count(*) FROM schema.result where type='publication'</entry> |
|
26 |
<entry key="queries.3.cql.query">oaftype=result and deletedbyinference=false and resulttypeid=publication</entry> |
|
27 |
<entry key="queries.3.name">Publications</entry> |
|
28 |
<entry key="queries.3.id">pubs</entry> |
|
29 |
|
|
30 |
|
|
31 |
<entry key="queries.4.type">search</entry> |
|
32 |
<entry key="queries.4.sql"> SELECT count(*) FROM schema.result WHERE bestlicense='Open Access' and type='publication'</entry> |
|
33 |
<entry key="queries.4.cql.query">oaftype=result and deletedbyinference=false and resulttypeid=publication and resultbestlicenseid exact "OPEN"</entry> |
|
34 |
<entry key="queries.4.name">Open Access Publications</entry> |
|
35 |
<entry key="queries.4.id">oapubs</entry> |
|
36 |
|
|
37 |
<entry key="queries.5.type">search</entry> |
|
38 |
<entry key="queries.5.sql">SELECT count(*) FROM schema.result WHERE bestlicense='Closed Access' and type='publication'</entry> |
|
39 |
<entry key="queries.5.cql.query">oaftype=result and deletedbyinference=false and resulttypeid=publication and resultbestlicenseid exact "CLOSED"</entry> |
|
40 |
<entry key="queries.5.name"> Closed Access Publications</entry> |
|
41 |
<entry key="queries.5.id">noapubs</entry> |
|
42 |
|
|
43 |
<!--FP7--> |
|
44 |
<entry key="queries.6.type">search</entry> |
|
45 |
<entry key="queries.6.sql">SELECT count (distinct result_projects.id) FROM schema.result, schema.result_projects, schema.project WHERE result.result_projects = result_projects.id and type='publication' |
|
46 |
and result_projects.project = project.id and funding_lvl0 = 'FP7'</entry> |
|
47 |
<entry key="queries.6.cql.query">oaftype=result and deletedbyinference=false and resulttypeid=publication and relfundinglevel0_id = "ec__________::EC::FP7"</entry> |
|
48 |
<entry key="queries.6.name">FP7 Publications</entry> |
|
49 |
<entry key="queries.6.id">fp7pubstotal</entry> |
|
50 |
|
|
51 |
<!-- ignore duplicate --> |
|
52 |
<entry key="queries.7.type">browse</entry> |
|
53 |
<entry key="queries.7.sql">SELECT count (distinct result_projects.id) FROM schema.result, schema.result_projects, schema.project WHERE result.result_projects = result_projects.id and type='publication' and result_projects.project = project.id and funding_lvl0 = 'FP7'</entry> |
|
54 |
<entry key="queries.7.cql.query"></entry> |
|
55 |
<entry key="queries.7.name"></entry> |
|
56 |
<entry key="queries.7.id"></entry> |
|
57 |
<!-- ignore duplicate --> |
|
58 |
|
|
59 |
<entry key="queries.8.type">search</entry> |
|
60 |
<entry key="queries.8.sql">SELECT count (distinct result_projects.id) FROM schema.result, schema.result_projects, schema.project WHERE result.result_projects = result_projects.id AND result_projects.project = project.id AND funding_lvl0 = 'FP7' AND bestlicense='Closed Access' AND type='publication'</entry> |
|
61 |
<entry key="queries.8.cql.query">oaftype=result and deletedbyinference=false and resulttypeid=publication and relfundinglevel0_id = "ec__________::EC::FP7" and resultbestlicenseid exact "CLOSED" </entry> |
|
62 |
<entry key="queries.8.name">FP7 Closed Access Publications</entry> |
|
63 |
<entry key="queries.8.id"> </entry> |
|
64 |
<entry key="queries.8.id">fp7noapubs</entry> |
|
65 |
|
|
66 |
<entry key="queries.9.type">search</entry> |
|
67 |
<entry key="queries.9.sql"> SELECT count(distinct result_projects.id) FROM schema.result, schema.result_projects, schema.project WHERE result_projects = result_projects.id AND result_projects.project = project.id and type='publication' and funding_lvl0 = 'FP7' and bestlicense='Open Access'</entry> |
|
68 |
<entry key="queries.9.cql.query">oaftype=result and deletedbyinference=false and resulttypeid=publication and relfundinglevel0_id = "ec__________::EC::FP7" and resultbestlicenseid exact "OPEN" </entry> |
|
69 |
<entry key="queries.9.name">FP7 Open Access Publications </entry> |
|
70 |
<entry key="queries.9.id">fp7oapubs</entry> |
|
71 |
|
|
72 |
<entry key="queries.10.type">search</entry> |
|
73 |
<entry key="queries.10.sql"> SELECT count(distinct result_projects.id) FROM schema.result, schema.result_projects, schema.project |
|
74 |
WHERE result.result_projects=result_projects.id AND result_projects.project = project.id and funding_lvl0 = 'FP7' and bestlicense='Restricted' and type='publication';</entry> |
|
75 |
<entry key="queries.10.cql.query">oaftype=result and deletedbyinference=false and resulttypeid=publication and relfundinglevel0_id = "ec__________::EC::FP7" and resultbestlicenseid exact "RESTRICTED" </entry> |
|
76 |
<entry key="queries.10.name">FP7 Restricted Access Publications </entry> |
|
77 |
<entry key="queries.10.id">fp7respubs </entry> |
|
78 |
|
|
79 |
<entry key="queries.11.type">search</entry> |
|
80 |
<entry key="queries.11.sql">SELECT count(distinct result_projects.id) FROM schema.result, schema.result_projects, schema.project |
|
81 |
WHERE result.result_projects = result_projects.id AND result_projects.project = project.id and funding_lvl0 = 'FP7' and bestlicense='Embargo' and type='publication' </entry> |
|
82 |
<entry key="queries.11.cql.query">oaftype=result and deletedbyinference=false and resulttypeid=publication and relfundinglevel0_id = "ec__________::EC::FP7" and resultbestlicenseid exact "EMBARGO"</entry> |
|
83 |
<entry key="queries.11.name">FP7 Embargo Access Publications </entry> |
|
84 |
<entry key="queries.11.id">fp7embpubs</entry> |
|
85 |
|
|
86 |
<entry key="queries.12.type">search</entry> |
|
87 |
<entry key="queries.12.sql">SELECT count(distinct project.id) from schema.result, schema.result_projects, schema.project WHERE result.result_projects = result_projects.id and type='publication' and result_projects.project = project.id and funding_lvl0='FP7' ;</entry> |
|
88 |
<entry key="queries.12.cql.query">not available</entry> |
|
89 |
<entry key="queries.12.name">FP7 Projects with Publications </entry> |
|
90 |
<entry key="queries.12.id">fp7projpubs</entry> |
|
91 |
|
|
92 |
<entry key="queries.13.type">search</entry> |
|
93 |
<entry key="queries.13.sql"> SELECT count(id) FROM schema.project WHERE funding_lvl0 = 'FP7';</entry> |
|
94 |
<entry key="queries.13.cql.query">oaftype=project and fundinglevel0_name=FP7</entry> |
|
95 |
<entry key="queries.13.name">FP7 Projects </entry> |
|
96 |
<entry key="queries.13.id">fp7projtotal</entry> |
|
97 |
|
|
98 |
<entry key="queries.14.type">search</entry> |
|
99 |
<entry key="queries.14.sql"> SELECT count(number) from schema.project where funding_lvl0='FP7' and sc39='yes'; </entry> |
|
100 |
<entry key="queries.14.cql.query">oaftype=project and fundinglevel0_name=FP7 and projectecsc39=true</entry> |
|
101 |
<entry key="queries.14.name"> FP7 Projects with SC39 </entry> |
|
102 |
<entry key="queries.14.id">sc39fp7projtotal</entry> |
|
103 |
|
|
104 |
<entry key="queries.15.type">search</entry> |
|
105 |
<entry key="queries.15.sql"> SELECT count(distinct project.id) FROM schema.result, schema.result_projects, schema.project WHERE result_projects.project=project.id and funding_lvl0 = 'FP7' and sc39='yes' and result.result_projects = result_projects.id and type='publication';</entry> |
|
106 |
<entry key="queries.15.cql.query">not available</entry> |
|
107 |
<entry key="queries.15.name">FP7 Projects with SC39 that have publications </entry> |
|
108 |
<entry key="queries.15.id">sc39fp7projpubs</entry> |
|
109 |
|
|
110 |
<entry key="queries.16.type">search</entry> |
|
111 |
<entry key="queries.16.sql"> SELECT count(distinct result_projects.id) FROM schema.result, schema.result_projects, schema.project |
|
112 |
WHERE result.result_projects = result_projects.id AND result_projects.project = project.id and funding_lvl0 = 'FP7' and bestlicense='Open Access' and sc39='yes' and type='publication';</entry> |
|
113 |
<entry key="queries.16.cql.query">not available</entry> |
|
114 |
<entry key="queries.16.name"> OA publications in FP7 Projects with SC39 </entry> |
|
115 |
<entry key="queries.16.id">sc39fp7oapubs</entry> |
|
116 |
|
|
117 |
<!-- WT --> |
|
118 |
|
|
119 |
<entry key="queries.17.type">search</entry> |
|
120 |
<entry key="queries.17.sql"> SELECT count(id) FROM schema.project WHERE funder = 'Wellcome Trust';</entry> |
|
121 |
<entry key="queries.17.cql.query">oaftype=project and fundername exact Wellcome Trust </entry> |
|
122 |
<entry key="queries.17.name">WT Projects </entry> |
|
123 |
<entry key="queries.17.id">wtprojtotal</entry> |
|
124 |
|
|
125 |
<entry key="queries.18.type">search</entry> |
|
126 |
<entry key="queries.18.sql"> SELECT count(distinct project.id) FROM schema.result, schema.project, schema.result_projects |
|
127 |
where result_projects.project = project.id and funder = 'Wellcome Trust' and result.result_projects = result_projects.id and type='publication';</entry> |
|
128 |
<entry key="queries.18.cql.query">not available</entry> |
|
129 |
<entry key="queries.18.name">WT Projects with Publications</entry> |
|
130 |
<entry key="queries.18.id">wtprojpubs</entry> |
|
131 |
|
|
132 |
<entry key="queries.19.type">search</entry> |
|
133 |
<entry key="queries.19.sql"> SELECT count(distinct result_projects.id) FROM schema.result, schema.result_projects, schema.project WHERE result.result_projects= result_projects.id AND result_projects.project = project.id and funder = 'Wellcome Trust' and type='publication';</entry> |
|
134 |
<entry key="queries.19.cql.query">oaftype=result and deletedbyinference=false and resulttypeid=publication and relfunderid exact "wt__________::WT" </entry> |
|
135 |
<entry key="queries.19.name"> Publications in WT Projects </entry> |
|
136 |
<entry key="queries.19.id">wtpubs</entry> |
|
137 |
|
|
138 |
<entry key="queries.20.type">search</entry> |
|
139 |
<entry key="queries.20.sql"> SELECT count(distinct result_projects.id) FROM schema.result, schema.result_projects, schema.project WHERE result.result_projects= result_projects.id |
|
140 |
AND result_projects.project = project.id and funder = 'Wellcome Trust' and bestlicense='Open Access' and type='publication';</entry> |
|
141 |
<entry key="queries.20.cql.query">oaftype=result and deletedbyinference=false and resulttypeid=publication and relfunderid exact "wt__________::WT" and resultbestlicenseid exact "OPEN"</entry> |
|
142 |
<entry key="queries.20.name"> Open Access Publications in WT Projects </entry> |
|
143 |
<entry key="queries.20.id">wtoapubs</entry> |
|
144 |
|
|
145 |
<entry key="queries.21.type">search</entry> |
|
146 |
<entry key="queries.21.sql"> SELECT count(distinct result_projects.id) FROM schema.result, schema.result_projects, schema.project WHERE result.result_projects = result_projects.id AND |
|
147 |
result_projects.project = project.id and funder = 'Wellcome Trust' and bestlicense='Restricted' and type='publication';</entry> |
|
148 |
<entry key="queries.21.cql.query"> oaftype=result and deletedbyinference=false and resulttypeid=publication and relfunderid exact "wt__________::WT" and resultbestlicenseid exact "RESTRICTED"</entry> |
|
149 |
<entry key="queries.21.name">Restricted Access Publications in WT Projects </entry> |
|
150 |
<entry key="queries.21.id">wtrespubs</entry> |
|
151 |
|
|
152 |
<entry key="queries.22.type">search</entry> |
|
153 |
<entry key="queries.22.sql"> SELECT count(distinct result_projects.id) FROM schema.result, schema.result_projects, schema.project |
|
154 |
WHERE result.result_projects=result_projects.id AND result_projects.project = project.id and funder = 'Wellcome Trust' and bestlicense='Embargo' and type='publication';</entry> |
|
155 |
<entry key="queries.22.cql.query"> oaftype=result and deletedbyinference=false and resulttypeid=publication and relfunderid exact "wt__________::WT" and resultbestlicenseid exact "EMBARGO"</entry> |
|
156 |
<entry key="queries.22.name">Embargo Publications in WT Projects</entry> |
|
157 |
<entry key="queries.22.id">wtembpubs</entry> |
|
158 |
|
|
159 |
<!--ERC--> |
|
160 |
|
|
161 |
<entry key="queries.23.type">search</entry> |
|
162 |
<entry key="queries.23.sql">SELECT count(id) FROM schema.project WHERE funding_lvl2 = 'ERC';</entry> |
|
163 |
<entry key="queries.23.cql.query">oaftype=project and fundinglevel2_name exact "ERC"</entry> |
|
164 |
<entry key="queries.23.name"> Projects with ERC funding </entry> |
|
165 |
<entry key="queries.23.id">ercprojtotal</entry> |
|
166 |
|
|
167 |
<entry key="queries.24.type">search</entry> |
|
168 |
<entry key="queries.24.sql">SELECT count(distinct project.id) FROM schema.result, schema.project, schema.result_projects |
|
169 |
where result_projects.project = project.id and project.funding_lvl2='ERC' and result.result_projects = result_projects.id and type='publication';</entry> |
|
170 |
<entry key="queries.24.cql.query">not available</entry> |
|
171 |
<entry key="queries.24.name"> ERC Projects with Publications</entry> |
|
172 |
<entry key="queries.24.id">ercprojpubs</entry> |
|
173 |
|
|
174 |
<entry key="queries.25.type">search</entry> |
|
175 |
<entry key="queries.25.sql">SELECT count(distinct result_projects.id) FROM schema.result, |
|
176 |
schema.result_projects, schema.project WHERE result.result_projects= result_projects.id |
|
177 |
AND result_projects.project = project.id and funding_lvl2 = 'ERC' and bestlicense='Open Access' and type='publication';</entry> |
|
178 |
<entry key="queries.25.cql.query">resulttypeid=publication and deletedbyinference=false and conceptname=ERC and resultbestlicenseid=OPEN</entry> |
|
179 |
<entry key="queries.25.name">Open Access Publications in ERC Projects </entry> |
|
180 |
<entry key="queries.25.id">ercoapubs</entry> |
|
181 |
|
|
182 |
<entry key="queries.26.type">search</entry> |
|
183 |
<entry key="queries.26.sql"> SELECT count(distinct result_projects.id) FROM schema.result, schema.result_projects, schema.project WHERE |
|
184 |
result.result_projects = result_projects.id AND result_projects.project = project.id and funding_lvl2 = 'ERC' and bestlicense='Restricted' and type='publication'; </entry> |
|
185 |
<entry key="queries.26.cql.query">resulttypeid=publication and deletedbyinference=false and conceptname=ERC and resultbestlicenseid=RESTRICTED</entry> |
|
186 |
<entry key="queries.26.name"> Restricted Publications in ERC Projects </entry> |
|
187 |
<entry key="queries.26.id">ercrespubs</entry> |
|
188 |
|
|
189 |
<entry key="queries.27.type">search</entry> |
|
190 |
<entry key="queries.27.sql"> SELECT count(distinct result_projects.id) FROM schema.result, schema.result_projects, schema.project |
|
191 |
WHERE result.result_projects=result_projects.id AND result_projects.project = project.id and funding_lvl2 = 'ERC' and bestlicense='Embargo' and type='publication'; </entry> |
|
192 |
<entry key="queries.27.cql.query">resulttypeid=publication and deletedbyinference=false and conceptname=ERC and resultbestlicenseid=EMBARGO</entry> |
|
193 |
<entry key="queries.27.name">Embargo Publications in ERC Projects </entry> |
|
194 |
<entry key="queries.27.id">ercembpubs</entry> |
|
195 |
|
|
196 |
<entry key="queries.28.type">search</entry> |
|
197 |
<entry key="queries.28.sql"> SELECT count(distinct result_projects.id) FROM schema.result, schema.project, schema.result_projects where result_projects.project = project.id and project.funding_lvl2='ERC' and result.result_projects = result_projects.id and type='publication'; </entry> |
|
198 |
<entry key="queries.28.cql.query">resulttypeid=publication and deletedbyinference=false and conceptname=ERC</entry> |
|
199 |
<entry key="queries.28.name">Total Publications in ERC Projects</entry> |
|
200 |
<entry key="queries.28.id">ercpubs</entry> |
|
201 |
|
|
202 |
|
|
203 |
<!--DATASOURCES--> |
|
204 |
|
|
205 |
<entry key="queries.29.type">search</entry> |
|
206 |
<entry key="queries.29.sql">SELECT count(*) FROM schema.datasource</entry> |
|
207 |
<entry key="queries.29.cql.query">oaftype=datasource </entry> |
|
208 |
<entry key="queries.29.name">Total Datasources </entry> |
|
209 |
<entry key="queries.29.id">datasrc</entry> |
|
210 |
|
|
211 |
<entry key="queries.30.type">search</entry> |
|
212 |
<entry key="queries.30.sql">select count(*) from schema.datasource where compatibility != 'not available' and compatibility != 'under validation'</entry> |
|
213 |
<entry key="queries.30.cql.query"> oaftype=datasource and datasourcecompatibilityid <> UNKNOWN and datasourcecompatibilityid <> notCompatible</entry> |
|
214 |
<entry key="queries.30.name">Valid Datasources</entry> |
|
215 |
<entry key="queries.30.id">dtsrcpubs</entry> |
|
216 |
|
|
217 |
<entry key="queries.31.type">search</entry> |
|
218 |
<entry key="queries.31.sql"> SELECT count(distinct rd.datasource) from schema.result_datasources rd ;</entry> |
|
219 |
<entry key="queries.31.cql.query">not available</entry> |
|
220 |
<entry key="queries.31.name">Datasources with Publications</entry> |
|
221 |
<entry key="queries.31.id">datasrc_withpubs</entry> |
|
222 |
|
|
223 |
<entry key="queries.32.type">search</entry> |
|
224 |
<entry key="queries.32.sql">select count(*) from schema.datasource where type='Journal Platform';</entry> |
|
225 |
<entry key="queries.32.cql.query">oaftype=datasource and datasourcetypeid=pubsrepository::journal</entry> |
|
226 |
<entry key="queries.32.name">Journal Datasources</entry> |
|
227 |
<entry key="queries.32.id">datasrc_journals</entry> |
|
228 |
|
|
229 |
<entry key="queries.33.type">search</entry> |
|
230 |
<entry key="queries.33.sql">select count(*) from schema.datasource where type='Publication Repository';</entry> |
|
231 |
<entry key="queries.33.cql.query">oaftype=datasource and datasourcetypeid=pubsrepository::unknown</entry> |
|
232 |
<entry key="queries.33.name">Publication Repository Datasources</entry> |
|
233 |
<entry key="queries.33.id">datasrc_pubrepo</entry> |
|
234 |
|
|
235 |
<entry key="queries.34.type">search</entry> |
|
236 |
<entry key="queries.34.sql">select count(*) from schema.datasource where type='Data Repository';</entry> |
|
237 |
<entry key="queries.34.cql.query">oaftype=datasource and datasourcetypeid=datarepository::unknown</entry> |
|
238 |
<entry key="queries.34.name">Data Repository Datasources</entry> |
|
239 |
<entry key="queries.34.id">datasrc_datarepo</entry> |
|
240 |
|
|
241 |
<entry key="queries.35.type">search</entry> |
|
242 |
<entry key="queries.35.sql">select count(*) from schema.datasource where type like 'Aggregator%';</entry> |
|
243 |
<entry key="queries.35.cql.query">oaftype=datasource and datasourcetypeid=aggregator::pubsrepository::unknown OR datasourcetypeid=aggregator::pubsrepository::journals OR datasourcetypeid=aggregator::datarepository</entry> |
|
244 |
<entry key="queries.35.name">Aggregator Datasources </entry> |
|
245 |
<entry key="queries.35.id">datasrc_aggr</entry> |
|
246 |
|
|
247 |
<entry key="queries.36.type">search</entry> |
|
248 |
<entry key="queries.36.sql">select count(distinct id) from schema.result where type='dataset';</entry> |
|
249 |
<entry key="queries.36.cql.query">resulttypeid=dataset</entry> |
|
250 |
<entry key="queries.36.name"> Total number of datasets </entry> |
|
251 |
<entry key="queries.36.id">data_total</entry> |
|
252 |
|
|
253 |
<entry key="queries.37.type">search</entry> |
|
254 |
<entry key="queries.37.sql">SELECT count(distinct funder) FROM schema.project</entry> |
|
255 |
<entry key="queries.37.cql.query">not available</entry> |
|
256 |
<entry key="queries.37.name">Funders</entry> |
|
257 |
<entry key="queries.37.id">funders</entry> |
|
258 |
|
|
259 |
<entry key="queries.38.type">search</entry> |
|
260 |
<entry key="queries.38.sql"> select count(*) from (select dor.organization as organization from schema.datasource_organizations dor join schema.result_datasources rd on rd.datasource=dor.id join schema.result r on r.id=rd.id where r.type='publication' |
|
261 |
union select por.organization as organization from schema.project_organizations por join schema.result_projects rp on rp.project=por.id join |
|
262 |
schema.result r on r.id=rp.id where r.type='publication') as foo;</entry> |
|
263 |
<entry key="queries.38.cql.query">not available</entry> |
|
264 |
<entry key="queries.38.name">Organizations with Publications </entry> |
|
265 |
<entry key="queries.38.id">org_withpubs</entry> |
|
266 |
|
|
267 |
|
|
268 |
|
|
269 |
<!-- |
|
270 |
<entry key="queries.2.type">browse</entry> |
|
271 |
<entry key="queries.2.sql">select count(id), access_type from schema.result group by access_type order by count(id)</entry> |
|
272 |
<entry key="queries.2.cql.query">oaftype=result and deletedbyinference=false</entry> |
|
273 |
<entry key="queries.2.cql.groupby">access_mode</entry> |
|
274 |
<entry key="queries.2.name"> Access Level Modes </entry> |
|
275 |
<entry key="queries.2.id"> Access Level Modes </entry> |
|
276 |
--> |
|
277 |
</properties> |
modules/uoa-stats-service/trunk/src/main/resources/eu/dnetlib/data/statsmanager/springContext-statsManager.properties | ||
---|---|---|
1 |
services.statsManager.serviceName = uoa-stats-service |
|
2 |
services.statsManager.db.driverClassName = org.postgresql.Driver |
|
3 |
services.statsManager.db.url = jdbc:postgresql://duffy.di.uoa.gr:5432/stats |
|
4 |
services.statsManager.db.username = sqoop |
|
5 |
services.statsManager.db.password = sqoop |
|
6 |
services.statsManager.historySchema = "stats_history" |
|
7 |
services.statsManager.validationQueriesFile = classpath:/eu/dnetlib/data/statsmanager/validationQueries.xml |
|
8 |
services.statsManager.actions.refreshAll=cacheController?action=refreshAll |
|
9 |
services.statsManager.actions.refreshNums =cacheController?action=refreshNums |
|
10 |
services.statsManager.actions.refreshCharts =cacheController?action=refreshCharts |
|
11 |
services.statsManager.actions.promoteNums =cacheController?action=promoteNums |
|
12 |
services.statsManager.actions.promoteCharts =cacheController?action=promoteCharts |
|
13 |
services.statsManager.actions.promoteAll =cacheController?action=promoteAll |
|
14 |
services.statsManager.actions.backup =cacheController?action=backup |
|
15 |
services.statsManager.actions.migrate =cacheController?action=migrate |
|
16 |
services.statsManager.actions.restore =cacheController?action=restore |
|
17 |
|
|
18 |
services.statsManager.shadowSearchServiceURL=http://duffy.di.uoa.gr:8080/shadowSearch/services/searchWebService |
|
19 |
|
|
20 |
services.statsManager.actions.prodURL=https://beta.openaire.eu/stats/ |
|
21 |
services.statsManager.actions.testURL=https://test.openaire.eu/stats/ |
|
22 |
|
|
23 |
|
|
24 |
services.statsManager.testhttpsCredentials =eri.katsari:UDxG2ydA |
|
25 |
services.statsManager.prodhttpsCredentials = : |
|
26 |
|
|
27 |
#DOES NOT DELETE SERVICE PROFILE OR RE-DEPLOY |
|
28 |
services.registration.default.removeRegistration=false |
|
29 |
#Search Service |
|
30 |
#services.search.indexMode = solr |
|
31 |
#services.search.mdFormat = TMF |
|
32 |
#services.search.vocabulariesPath = /tmp/vocabularies/ |
|
33 |
|
|
34 |
|
|
35 |
services.search.infrastructure = shadow |
modules/uoa-stats-service/trunk/src/main/resources/eu/dnetlib/data/statsmanager/stats_history.sql | ||
---|---|---|
1 |
-- |
|
2 |
-- PostgreSQL database dump |
|
3 |
-- |
|
4 |
|
|
5 |
DROP SCHEMA IF EXISTS stats_history CASCADE; |
|
6 |
CREATE SCHEMA stats_history; |
|
7 |
ALTER SCHEMA stats_history OWNER TO sqoop; |
|
8 |
|
|
9 |
-- |
|
10 |
-- Name: query; Type: TABLE; Schema: public; Owner: postgres; Tablespace: |
|
11 |
-- |
|
12 |
|
|
13 |
CREATE TABLE stats_history.query ( |
|
14 |
query text, |
|
15 |
description text, |
|
16 |
query_id text |
|
17 |
); |
|
18 |
|
|
19 |
alter table stats_history.query owner to sqoop; |
|
20 |
-- |
|
21 |
-- Name: value; Type: TABLE; Schema: public; Owner: postgres; Tablespace: |
|
22 |
-- |
|
23 |
|
|
24 |
CREATE TABLE stats_history.value ( |
|
25 |
num text, |
|
26 |
date text, |
|
27 |
query_id text |
|
28 |
); |
|
29 |
|
|
30 |
alter table stats_history.value owner to sqoop; |
|
31 |
|
|
32 |
|
|
33 |
-- |
|
34 |
-- Name: public; Type: ACL; Schema: -; Owner: postgres |
|
35 |
-- |
|
36 |
|
|
37 |
--REVOKE ALL ON SCHEMA stats_history FROM PUBLIC; |
|
38 |
--REVOKE ALL ON SCHEMA stats_history FROM sqoop; |
|
39 |
--GRANT ALL ON SCHEMA stats_history TO sqoop; |
|
40 |
--GRANT ALL ON SCHEMA stats_history TO PUBLIC; |
|
41 |
|
|
42 |
|
|
43 |
-- |
|
44 |
-- PostgreSQL database dump complete |
|
45 |
-- |
|
46 |
|
|
47 |
|
modules/uoa-stats-service/trunk/src/main/resources/uoa-stats-service.properties | ||
---|---|---|
1 |
#Generated by ANT |
|
2 |
name=uoa-stats-service |
|
3 |
version=1.0.0 |
|
4 |
label= |
Also available in: Unified diff
dnet45