Project

General

Profile

« Previous | Next » 

Revision 58911

Added by Dimitris Pierrakos almost 4 years ago

Split json files

View differences:

modules/dnet-openaire-usage-stats-export-wf/branches/usage_stats_export_v2/dnet-openaire-usage-stats-export/src/main/java/eu/dnetlib/usagestats/export/ConnectDB.java
17 17

  
18 18
public abstract class ConnectDB {
19 19

  
20
    private static Connection DB_CONNECTION;
20
    public static Connection DB_CONNECTION;
21 21

  
22 22
    private static String dbURL;
23 23
    private static String dbUsername;
modules/dnet-openaire-usage-stats-export-wf/branches/usage_stats_export_v2/dnet-openaire-usage-stats-export/src/main/java/eu/dnetlib/usagestats/export/PiwikStatsDB.java
429 429
        sql = "DROP VIEW IF EXISTS  result_views_monthly_tmp";
430 430
        stmt.executeUpdate(sql);
431 431

  
432
        sql = "DROP TABLE IF EXISTS views_stats_tmp;";
433
        stmt.executeUpdate(sql);
434
        
435
        sql = "DROP TABLE IF EXISTS downloads_stats_tmp;";
436
        stmt.executeUpdate(sql);
437
        
438
        sql = "DROP TABLE IF EXISTS process_portal_log_tmp;";
439
        stmt.executeUpdate(sql);
440

  
432 441
        sql = "DROP TABLE IF EXISTS piwiklogtmp;";
433 442
        stmt.executeUpdate(sql);
434 443

  
......
565 574
        int batch_size = 0;
566 575
        JSONParser parser = new JSONParser();
567 576
        for (String jsonFile : jsonFiles) {
577
            System.out.println(jsonFile);
568 578
            JSONArray jsonArray = (JSONArray) parser.parse(readHDFSFile(jsonFile));
569 579

  
570 580
            for (Object aJsonArray : jsonArray) {
......
642 652

  
643 653
        String sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'oaItem\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.result_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
644 654
        stmt.executeUpdate(sql);
645
        stmt.close();
655
//        stmt.close();
646 656
//        con.commit();
647 657

  
648 658
        stmt = con.createStatement();
649 659
        sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'datasource\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.datasource_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
650 660
        stmt.executeUpdate(sql);
651
        stmt.close();
661
//        stmt.close();
652 662
//        con.commit();
653 663

  
654 664
        stmt = con.createStatement();
655 665
        sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'organization\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.organization_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
656 666
        stmt.executeUpdate(sql);
657
        stmt.close();
667
//        stmt.close();
658 668
//        con.commit();
659 669

  
660 670
        stmt = con.createStatement();
661 671
        sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'project\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.project_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
662 672
        stmt.executeUpdate(sql);
673
 //       stmt.close();
674
//        con.commit();
663 675
        stmt.close();
664
//        con.commit();
665

  
666
        con.close();
676
        ConnectDB.getConnection().commit();
677
        ConnectDB.getConnection().close();
667 678
    }
668 679

  
669 680
    private void cleanOAI() throws Exception {
......
894 905
        sql = "insert into pageviews_stats select * from pageviews_stats_tmp;";
895 906
        stmt.executeUpdate(sql);
896 907

  
897
        sql = "DROP TABLE IF EXISTS views_stats_tmp;";
898
        stmt.executeUpdate(sql);
899
        
900
        sql = "DROP TABLE IF EXISTS downloads_stats_tmp;";
901
        stmt.executeUpdate(sql);
902
        
903
        sql = "DROP TABLE IF EXISTS pageviews_stats_tmp;";
904
        stmt.executeUpdate(sql);
905

  
906
        sql = "DROP TABLE IF EXISTS process_portal_log_tmp;";
907
        stmt.executeUpdate(sql);
908
       
909
        
910 908
        stmt.close();
911 909
        ConnectDB.getConnection().commit();
912 910
        ConnectDB.getConnection().close();

Also available in: Unified diff