Revision 58437
Added by Dimitris Pierrakos about 4 years ago
PiwikStatsDB.java | ||
---|---|---|
285 | 285 |
sql = "CREATE TABLE IF NOT EXISTS views_stats_tmp AS SELECT 'OpenAIRE'::TEXT as source, d.id as repository_id, ro.id as result_id, month as date, max(views) AS count, max(openaire_referrer) AS openaire FROM result_views_monthly_tmp p, public.datasource d, public.result_oids ro where p.source!='5' AND p.source=d.piwik_id and p.id=ro.orid group by repository_id, result_id, date ORDER BY repository_id, result_id, date;"; |
286 | 286 |
stmt.executeUpdate(sql); |
287 | 287 |
|
288 |
sql = "CREATE TABLE IF NOT EXISTS views_stats (like views_stats_tmp including all)"; |
|
289 |
stmt.executeUpdate(sql); |
|
290 |
|
|
288 | 291 |
// sql = "SELECT 'OpenAIRE'::TEXT as source, d.id as repository_id, ro.id as result_id, month as date, max(views) AS count INTO pageviews_stats FROM result_views_monthly p, datasource d, result_oids ro where p.source='5' AND p.source=d.piwik_id and p.id=ro.orid group by repository_id, result_id, date ORDER BY repository_id, result_id, date;"; |
289 |
sql = "CREATE TABLE IF NOT EXISTS pageviews_stats_tmp AS SELECT 'OpenAIRE'::TEXT as source, d.id as repository_id, ro.id as result_id, month as date, max(views) AS count FROM result_views_monthly_tmp p, public.datasource d, public.result_oids ro where p.source='5' AND p.source=d.piwik_id and p.id=ro.orid group by repository_id, result_id, date ORDER BY repository_id, result_id, date;";
|
|
292 |
sql = "CREATE TABLE IF NOT EXISTS pageviews_stats_tmp AS SELECT 'OpenAIRE'::TEXT as source, d.id as repository_id, ro.id as result_id, month as date, max(views) AS count FROM result_views_monthly_tmp p, public.datasource d, public.result_oids ro where p.source='109' AND p.source=d.piwik_id and p.id=ro.orid group by repository_id, result_id, date ORDER BY repository_id, result_id, date;";
|
|
290 | 293 |
stmt.executeUpdate(sql); |
291 | 294 |
|
292 |
sql = "DROP VIEW IF EXISTS result_views_monthly_tmp;";
|
|
295 |
sql = "CREATE TABLE IF NOT EXISTS pageviews_stats (like pageviews_stats_tmp including all)";
|
|
293 | 296 |
stmt.executeUpdate(sql); |
294 |
|
|
297 |
|
|
295 | 298 |
stmt.close(); |
296 | 299 |
ConnectDB.getConnection().commit(); |
297 | 300 |
ConnectDB.getConnection().close(); |
... | ... | |
325 | 328 |
Statement stmt = ConnectDB.getConnection().createStatement(); |
326 | 329 |
ConnectDB.getConnection().setAutoCommit(false); |
327 | 330 |
|
328 |
//String sql = "CREATE OR REPLACE VIEW result_downloads_monthly as select entity_id AS id, COUNT(entity_id) as downloads, extract('year' from timestamp::date) ||'/'|| LPAD(CAST(extract('month' from timestamp::date) AS VARCHAR), 2, '0') AS month, source FROM piwiklog where action='download' and (source_item_type='oaItem' or source_item_type='repItem') group by id, month, source order by source, id, month;"; |
|
331 |
//String sql = "CREATE OR REPLACE VIEW result_downloads_monthly as select entity_id AS id, COUNT(entity_id) as downloads, extract('year' from timestamp::date) ||'/'|| LPAD(CAST(extract('month' from timestamp::date) AS VARCHAR), 2, '0') AS month, source FROM piwiklog where action='download' and (source_item_type='oaItem' or source_item_type='repItem') group by id, month, source order by source, id, month;";
|
|
329 | 332 |
String sql = "CREATE OR REPLACE VIEW result_downloads_monthly_tmp as select entity_id AS id, COUNT(entity_id) as downloads, SUM(CASE WHEN referrer_name LIKE '%openaire%' THEN 1 ELSE 0 END) AS openaire_referrer, extract('year' from timestamp::date) ||'/'|| LPAD(CAST(extract('month' from timestamp::date) AS VARCHAR), 2, '0') AS month, source FROM piwiklogtmp where action='download' and (source_item_type='oaItem' or source_item_type='repItem') group by id, month, source order by source, id, month;"; |
330 | 333 |
stmt.executeUpdate(sql); |
331 | 334 |
|
... | ... | |
333 | 336 |
// sql = "SELECT 'OpenAIRE'::TEXT as source, d.id as repository_id, ro.id as result_id, month as date, max(downloads) AS count, max(openaire_referrer) AS openaire INTO downloads_stats FROM result_downloads_monthly p, datasource d, result_oids ro where p.source!='5' AND p.source=d.piwik_id and p.id=ro.orid group by repository_id, result_id, date ORDER BY repository_id, result_id, date;"; |
334 | 337 |
sql = "CREATE TABLE IF NOT EXISTS downloads_stats_tmp AS SELECT 'OpenAIRE'::TEXT as source, d.id as repository_id, ro.id as result_id, month as date, max(downloads) AS count, max(openaire_referrer) AS openaire FROM result_downloads_monthly_tmp p, public.datasource d, public.result_oids ro where p.source=d.piwik_id and p.id=ro.orid group by repository_id, result_id, date ORDER BY repository_id, result_id, date;"; |
335 | 338 |
stmt.executeUpdate(sql); |
336 |
|
|
339 |
|
|
340 |
sql = "CREATE TABLE IF NOT EXISTS downloads_stats (like downloads_stats_tmp including all)"; |
|
341 |
stmt.executeUpdate(sql); |
|
342 |
|
|
337 | 343 |
sql = "DROP VIEW IF EXISTS result_downloads_monthly_tmp;"; |
338 | 344 |
stmt.executeUpdate(sql); |
339 | 345 |
|
... | ... | |
395 | 401 |
sql = "CREATE INDEX IF NOT EXISTS downloads_stats_date ON downloads_stats USING btree(date);"; |
396 | 402 |
stmt.executeUpdate(sql); |
397 | 403 |
|
404 |
// sql = "SELECT coalesce(ds.source, vs.source) as source, coalesce(ds.repository_id, vs.repository_id) as repository_id, coalesce(ds.result_id, vs.result_id) as result_id, coalesce(ds.date, vs.date) as date, coalesce(ds.count, 0) as downloads, coalesce(vs.count, 0) as views, coalesce(ds.openaire, 0) as openaire_downloads, coalesce(vs.openaire, 0) as openaire_views INTO usage_stats FROM downloads_stats AS ds FULL OUTER JOIN views_stats AS vs ON ds.source=vs.source AND ds.repository_id=vs.repository_id AND ds.result_id=vs.result_id AND ds.date=vs.date;"; |
|
398 | 405 |
sql = "CREATE TABLE IF NOT EXISTS usage_stats AS SELECT coalesce(ds.source, vs.source) as source, coalesce(ds.repository_id, vs.repository_id) as repository_id, coalesce(ds.result_id, vs.result_id) as result_id, coalesce(ds.date, vs.date) as date, coalesce(ds.count, 0) as downloads, coalesce(vs.count, 0) as views, coalesce(ds.openaire, 0) as openaire_downloads, coalesce(vs.openaire, 0) as openaire_views FROM downloads_stats AS ds FULL OUTER JOIN views_stats AS vs ON ds.source=vs.source AND ds.repository_id=vs.repository_id AND ds.result_id=vs.result_id AND ds.date=vs.date;"; |
399 | 406 |
stmt.executeUpdate(sql); |
400 | 407 |
|
... | ... | |
409 | 416 |
|
410 | 417 |
sql = "CREATE INDEX IF NOT EXISTS usage_stats_date ON usage_stats USING btree(date);"; |
411 | 418 |
stmt.executeUpdate(sql); |
419 |
|
|
420 |
sql = "DROP TABLE IF EXISTS process_portal_log_tmp;"; |
|
421 |
stmt.executeUpdate(sql); |
|
412 | 422 |
|
423 |
sql = "DROP TABLE IF EXISTS pageviews_stats_tmp;"; |
|
424 |
stmt.executeUpdate(sql); |
|
425 |
|
|
426 |
sql = "DROP VIEW IF EXISTS result_views_monthly_tmp"; |
|
427 |
stmt.executeUpdate(sql); |
|
428 |
|
|
429 |
sql = "DROP TABLE IF EXISTS piwiklogtmp;"; |
|
430 |
stmt.executeUpdate(sql); |
|
431 |
|
|
432 |
sql = "DROP TABLE IF EXISTS sushilogtmp;"; |
|
433 |
stmt.executeUpdate(sql); |
|
434 |
|
|
413 | 435 |
stmt.close(); |
414 | 436 |
ConnectDB.getConnection().commit(); |
415 | 437 |
ConnectDB.getConnection().close(); |
... | ... | |
615 | 637 |
Statement stmt = con.createStatement(); |
616 | 638 |
con.setAutoCommit(false); |
617 | 639 |
|
618 |
String sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'oaItem\', timestamp, referrer_name, agent FROM process_portal_log_tmp, result_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;"; |
|
640 |
String sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'oaItem\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.result_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
|
|
619 | 641 |
stmt.executeUpdate(sql); |
620 | 642 |
stmt.close(); |
621 | 643 |
// con.commit(); |
... | ... | |
627 | 649 |
// con.commit(); |
628 | 650 |
|
629 | 651 |
stmt = con.createStatement(); |
630 |
sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'organization\', timestamp, referrer_name, agent FROM process_portal_log_tmp, organization_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;"; |
|
652 |
sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'organization\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.organization_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
|
|
631 | 653 |
stmt.executeUpdate(sql); |
632 | 654 |
stmt.close(); |
633 | 655 |
// con.commit(); |
634 | 656 |
|
635 | 657 |
stmt = con.createStatement(); |
636 |
sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'project\', timestamp, referrer_name, agent FROM process_portal_log_tmp, project_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;"; |
|
658 |
sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'project\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.project_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
|
|
637 | 659 |
stmt.executeUpdate(sql); |
638 | 660 |
stmt.close(); |
639 | 661 |
// con.commit(); |
... | ... | |
856 | 878 |
private void updateProdTables() throws SQLException { |
857 | 879 |
Statement stmt = ConnectDB.getConnection().createStatement(); |
858 | 880 |
ConnectDB.getConnection().setAutoCommit(false); |
859 |
String sql = "insert into piwiklog select * from piwiklogtmp;"; |
|
860 |
stmt.executeUpdate(sql); |
|
881 |
|
|
882 |
String sql = "insert into piwiklog select * from piwiklogtmp;"; |
|
883 |
stmt.executeUpdate(sql); |
|
861 | 884 |
|
862 | 885 |
sql = "insert into views_stats select * from views_stats_tmp;"; |
863 | 886 |
stmt.executeUpdate(sql); |
... | ... | |
868 | 891 |
sql = "insert into pageviews_stats select * from pageviews_stats_tmp;"; |
869 | 892 |
stmt.executeUpdate(sql); |
870 | 893 |
|
871 |
sql = "DROP TABLE IF EXISTS piwiklogtmp;"; |
|
872 |
stmt.executeUpdate(sql); |
|
873 |
|
|
874 | 894 |
sql = "DROP TABLE IF EXISTS views_stats_tmp;"; |
875 | 895 |
stmt.executeUpdate(sql); |
876 | 896 |
|
877 | 897 |
sql = "DROP TABLE IF EXISTS downloads_stats_tmp;"; |
878 | 898 |
stmt.executeUpdate(sql); |
879 | 899 |
|
880 |
sql = "DROP TABLE IF EXISTS pageviews_stats_stats_tmp;";
|
|
900 |
sql = "DROP TABLE IF EXISTS pageviews_stats_tmp;"; |
|
881 | 901 |
stmt.executeUpdate(sql); |
902 |
|
|
903 |
sql = "DROP TABLE IF EXISTS process_portal_log_tmp;"; |
|
904 |
stmt.executeUpdate(sql); |
|
905 |
|
|
882 | 906 |
|
883 |
|
|
884 | 907 |
stmt.close(); |
885 | 908 |
ConnectDB.getConnection().commit(); |
886 | 909 |
ConnectDB.getConnection().close(); |
Also available in: Unified diff
New Workflow for Usage Stats