Project

General

Profile

« Previous | Next » 

Revision 58437

Added by Dimitris Pierrakos about 4 years ago

New Workflow for Usage Stats

View differences:

PiwikStatsDB.java
285 285
        sql = "CREATE TABLE IF NOT EXISTS views_stats_tmp AS SELECT 'OpenAIRE'::TEXT as source, d.id as repository_id, ro.id as result_id, month as date, max(views) AS count, max(openaire_referrer) AS openaire FROM result_views_monthly_tmp p, public.datasource d, public.result_oids ro where p.source!='5' AND p.source=d.piwik_id and p.id=ro.orid group by repository_id, result_id, date ORDER BY repository_id, result_id, date;";
286 286
        stmt.executeUpdate(sql);
287 287

  
288
        sql = "CREATE TABLE IF NOT EXISTS views_stats (like views_stats_tmp including all)";
289
        stmt.executeUpdate(sql);
290

  
288 291
//        sql = "SELECT 'OpenAIRE'::TEXT as source, d.id as repository_id, ro.id as result_id, month as date, max(views) AS count INTO pageviews_stats FROM result_views_monthly p, datasource d, result_oids ro where p.source='5' AND p.source=d.piwik_id and p.id=ro.orid group by repository_id, result_id, date ORDER BY repository_id, result_id, date;";
289
        sql = "CREATE TABLE IF NOT EXISTS pageviews_stats_tmp AS SELECT 'OpenAIRE'::TEXT as source, d.id as repository_id, ro.id as result_id, month as date, max(views) AS count FROM result_views_monthly_tmp p, public.datasource d, public.result_oids ro where p.source='5' AND p.source=d.piwik_id and p.id=ro.orid group by repository_id, result_id, date ORDER BY repository_id, result_id, date;";
292
        sql = "CREATE TABLE IF NOT EXISTS pageviews_stats_tmp AS SELECT 'OpenAIRE'::TEXT as source, d.id as repository_id, ro.id as result_id, month as date, max(views) AS count FROM result_views_monthly_tmp p, public.datasource d, public.result_oids ro where p.source='109' AND p.source=d.piwik_id and p.id=ro.orid group by repository_id, result_id, date ORDER BY repository_id, result_id, date;";
290 293
        stmt.executeUpdate(sql);
291 294

  
292
        sql = "DROP VIEW IF EXISTS result_views_monthly_tmp;";
295
        sql = "CREATE TABLE IF NOT EXISTS pageviews_stats (like pageviews_stats_tmp including all)";
293 296
        stmt.executeUpdate(sql);
294

  
297
        
295 298
        stmt.close();
296 299
        ConnectDB.getConnection().commit();
297 300
        ConnectDB.getConnection().close();
......
325 328
        Statement stmt = ConnectDB.getConnection().createStatement();
326 329
        ConnectDB.getConnection().setAutoCommit(false);
327 330

  
328
        //String sql = "CREATE OR REPLACE VIEW result_downloads_monthly as select entity_id AS id, COUNT(entity_id) as downloads, extract('year' from timestamp::date) ||'/'|| LPAD(CAST(extract('month' from timestamp::date) AS VARCHAR), 2, '0') AS month, source FROM piwiklog where action='download' and (source_item_type='oaItem' or source_item_type='repItem') group by id, month, source order by source, id, month;";
331
         //String sql = "CREATE OR REPLACE VIEW result_downloads_monthly as select entity_id AS id, COUNT(entity_id) as downloads, extract('year' from timestamp::date) ||'/'|| LPAD(CAST(extract('month' from timestamp::date) AS VARCHAR), 2, '0') AS month, source FROM piwiklog where action='download' and (source_item_type='oaItem' or source_item_type='repItem') group by id, month, source order by source, id, month;";
329 332
        String sql = "CREATE OR REPLACE VIEW result_downloads_monthly_tmp as select entity_id AS id, COUNT(entity_id) as downloads, SUM(CASE WHEN referrer_name LIKE '%openaire%' THEN 1 ELSE 0 END) AS openaire_referrer, extract('year' from timestamp::date) ||'/'|| LPAD(CAST(extract('month' from timestamp::date) AS VARCHAR), 2, '0') AS month, source FROM piwiklogtmp where action='download' and (source_item_type='oaItem' or source_item_type='repItem') group by id, month, source order by source, id, month;";
330 333
        stmt.executeUpdate(sql);
331 334

  
......
333 336
//        sql = "SELECT 'OpenAIRE'::TEXT as source, d.id as repository_id, ro.id as result_id, month as date, max(downloads) AS count, max(openaire_referrer) AS openaire INTO downloads_stats FROM result_downloads_monthly p, datasource d, result_oids ro where p.source!='5' AND p.source=d.piwik_id and p.id=ro.orid group by repository_id, result_id, date ORDER BY repository_id, result_id, date;";
334 337
        sql = "CREATE TABLE IF NOT EXISTS downloads_stats_tmp AS SELECT 'OpenAIRE'::TEXT as source, d.id as repository_id, ro.id as result_id, month as date, max(downloads) AS count, max(openaire_referrer) AS openaire FROM result_downloads_monthly_tmp p, public.datasource d, public.result_oids ro where p.source=d.piwik_id and p.id=ro.orid group by repository_id, result_id, date ORDER BY repository_id, result_id, date;";
335 338
        stmt.executeUpdate(sql);
336

  
339
        
340
        sql = "CREATE TABLE IF NOT EXISTS downloads_stats (like downloads_stats_tmp including all)";
341
        stmt.executeUpdate(sql);
342
        
337 343
        sql = "DROP VIEW IF EXISTS result_downloads_monthly_tmp;";
338 344
        stmt.executeUpdate(sql);
339 345

  
......
395 401
        sql = "CREATE INDEX IF NOT EXISTS downloads_stats_date ON downloads_stats USING btree(date);";
396 402
        stmt.executeUpdate(sql);
397 403

  
404
//        sql = "SELECT coalesce(ds.source, vs.source) as source, coalesce(ds.repository_id, vs.repository_id) as repository_id, coalesce(ds.result_id, vs.result_id) as result_id, coalesce(ds.date, vs.date) as date, coalesce(ds.count, 0) as downloads, coalesce(vs.count, 0) as views, coalesce(ds.openaire, 0) as openaire_downloads, coalesce(vs.openaire, 0) as openaire_views INTO usage_stats FROM downloads_stats AS ds FULL OUTER JOIN views_stats AS vs ON ds.source=vs.source AND ds.repository_id=vs.repository_id AND ds.result_id=vs.result_id AND ds.date=vs.date;";
398 405
        sql = "CREATE TABLE IF NOT EXISTS usage_stats AS SELECT coalesce(ds.source, vs.source) as source, coalesce(ds.repository_id, vs.repository_id) as repository_id, coalesce(ds.result_id, vs.result_id) as result_id, coalesce(ds.date, vs.date) as date, coalesce(ds.count, 0) as downloads, coalesce(vs.count, 0) as views, coalesce(ds.openaire, 0) as openaire_downloads, coalesce(vs.openaire, 0) as openaire_views FROM downloads_stats AS ds FULL OUTER JOIN views_stats AS vs ON ds.source=vs.source AND ds.repository_id=vs.repository_id AND ds.result_id=vs.result_id AND ds.date=vs.date;";
399 406
        stmt.executeUpdate(sql);
400 407

  
......
409 416

  
410 417
        sql = "CREATE INDEX IF NOT EXISTS usage_stats_date ON usage_stats USING btree(date);";
411 418
        stmt.executeUpdate(sql);
419
       
420
        sql = "DROP TABLE IF EXISTS process_portal_log_tmp;";
421
        stmt.executeUpdate(sql);
412 422

  
423
        sql = "DROP TABLE IF EXISTS pageviews_stats_tmp;";
424
        stmt.executeUpdate(sql);
425
        
426
        sql = "DROP VIEW IF EXISTS  result_views_monthly_tmp";
427
        stmt.executeUpdate(sql);
428

  
429
        sql = "DROP TABLE IF EXISTS piwiklogtmp;";
430
        stmt.executeUpdate(sql);
431

  
432
        sql = "DROP TABLE IF EXISTS sushilogtmp;";
433
        stmt.executeUpdate(sql);
434

  
413 435
        stmt.close();
414 436
        ConnectDB.getConnection().commit();
415 437
        ConnectDB.getConnection().close();
......
615 637
        Statement stmt = con.createStatement();
616 638
        con.setAutoCommit(false);
617 639

  
618
        String sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'oaItem\', timestamp, referrer_name, agent FROM process_portal_log_tmp, result_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
640
        String sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'oaItem\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.result_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
619 641
        stmt.executeUpdate(sql);
620 642
        stmt.close();
621 643
//        con.commit();
......
627 649
//        con.commit();
628 650

  
629 651
        stmt = con.createStatement();
630
        sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'organization\', timestamp, referrer_name, agent FROM process_portal_log_tmp, organization_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
652
        sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'organization\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.organization_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
631 653
        stmt.executeUpdate(sql);
632 654
        stmt.close();
633 655
//        con.commit();
634 656

  
635 657
        stmt = con.createStatement();
636
        sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'project\', timestamp, referrer_name, agent FROM process_portal_log_tmp, project_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
658
        sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'project\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.project_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
637 659
        stmt.executeUpdate(sql);
638 660
        stmt.close();
639 661
//        con.commit();
......
856 878
    private void updateProdTables() throws SQLException {
857 879
        Statement stmt = ConnectDB.getConnection().createStatement();
858 880
        ConnectDB.getConnection().setAutoCommit(false);
859
        String sql = "insert into piwiklog select * from piwiklogtmp;";
860
        stmt.executeUpdate(sql);
881

  
882
       String sql = "insert into piwiklog select * from piwiklogtmp;";
883
       stmt.executeUpdate(sql);
861 884
        
862 885
        sql = "insert into views_stats select * from views_stats_tmp;";
863 886
        stmt.executeUpdate(sql);
......
868 891
        sql = "insert into pageviews_stats select * from pageviews_stats_tmp;";
869 892
        stmt.executeUpdate(sql);
870 893

  
871
        sql = "DROP TABLE IF EXISTS piwiklogtmp;";
872
        stmt.executeUpdate(sql);
873

  
874 894
        sql = "DROP TABLE IF EXISTS views_stats_tmp;";
875 895
        stmt.executeUpdate(sql);
876 896
        
877 897
        sql = "DROP TABLE IF EXISTS downloads_stats_tmp;";
878 898
        stmt.executeUpdate(sql);
879 899
        
880
        sql = "DROP TABLE IF EXISTS pageviews_stats_stats_tmp;";
900
        sql = "DROP TABLE IF EXISTS pageviews_stats_tmp;";
881 901
        stmt.executeUpdate(sql);
902

  
903
        sql = "DROP TABLE IF EXISTS process_portal_log_tmp;";
904
        stmt.executeUpdate(sql);
905
       
882 906
        
883
        
884 907
        stmt.close();
885 908
        ConnectDB.getConnection().commit();
886 909
        ConnectDB.getConnection().close();

Also available in: Unified diff